@rlyle1179/serverless-aws-batch 1.1.19 → 1.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +362 -362
- package/index.js +66 -66
- package/lib/awscli.js +107 -107
- package/lib/batchenvironment.js +418 -418
- package/lib/batchtask.js +390 -390
- package/lib/docker.js +139 -139
- package/lib/ecr.js +29 -29
- package/lib/generateCoreTemplate.js +120 -120
- package/lib/schedule.js +99 -90
- package/package.json +35 -35
package/lib/batchenvironment.js
CHANGED
|
@@ -1,418 +1,418 @@
|
|
|
1
|
-
const BbPromise = require('bluebird');
|
|
2
|
-
const _ = require('lodash');
|
|
3
|
-
const util = require('util');
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* @returns {string} "IamRoleBatchService"
|
|
7
|
-
*/
|
|
8
|
-
function getBatchServiceRoleLogicalId() {
|
|
9
|
-
return "IamRoleBatchService";
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
/**
|
|
13
|
-
* @returns {string} "IamRoleBatchInstanceManagement"
|
|
14
|
-
*/
|
|
15
|
-
function getBatchInstanceManagementRoleLogicalId() {
|
|
16
|
-
return "IamRoleBatchInstanceManagement";
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* @returns {string} "IamProfileBatchInstanceManagement"
|
|
21
|
-
*/
|
|
22
|
-
function getBatchInstanceManagementProfileLogicalId() {
|
|
23
|
-
return "IamProfileBatchInstanceManagement";
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* @returns {string} "IamRoleBatchSpotFleetManagement"
|
|
28
|
-
*/
|
|
29
|
-
function getBatchSpotFleetManagementRoleLogicalId() {
|
|
30
|
-
return "IamRoleBatchSpotFleetManagement";
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* @returns {string} "IamRoleLambdaScheduleExecution"
|
|
35
|
-
*/
|
|
36
|
-
function getLambdaScheduleExecutionRoleLogicalId() {
|
|
37
|
-
return "IamRoleLambdaScheduleExecution";
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* @returns {string} "BatchComputeEnvironment"
|
|
42
|
-
*/
|
|
43
|
-
function getBatchComputeEnvironmentLogicalId() {
|
|
44
|
-
return "BatchComputeEnvironment";
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* @returns {string} "BatchJobQueue"
|
|
49
|
-
*/
|
|
50
|
-
function getBatchJobQueueLogicalId() {
|
|
51
|
-
return "BatchJobQueue";
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* @returns {string} The name of the job queue to be used when submitting the job
|
|
56
|
-
*/
|
|
57
|
-
function getBatchJobQueueName() {
|
|
58
|
-
return `${this.provider.serverless.service.service}-${this.provider.getStage()}-JobQueue`;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Validates the "batch" object in the serverless config to ensure that we have:
|
|
63
|
-
* - subnets
|
|
64
|
-
* - securityGroups
|
|
65
|
-
*/
|
|
66
|
-
function validateAWSBatchServerlessConfig() {
|
|
67
|
-
const provider = this.serverless.service.provider;
|
|
68
|
-
if (! provider.hasOwnProperty("batch")) {
|
|
69
|
-
throw new Error("'batch' configuration not defined on the provider");
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
const batch = provider.batch;
|
|
73
|
-
if (! batch.hasOwnProperty("SecurityGroupIds")) {
|
|
74
|
-
throw new Error("'batch' configuration does not contain property 'SecurityGroupIds' (make sure it's capitalized)");
|
|
75
|
-
}
|
|
76
|
-
if (! batch.hasOwnProperty("Subnets")) {
|
|
77
|
-
throw new Error("'batch' configuration does not contain property 'Subnets' (make sure it's capitalized)");
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Generates the IAM Service Role Object to be used by the Batch Compute Environment
|
|
83
|
-
*/
|
|
84
|
-
function generateBatchServiceRole() {
|
|
85
|
-
const batchServiceRoleName = `BatchServiceRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
86
|
-
const batchServiceRoleTemplate = `
|
|
87
|
-
{
|
|
88
|
-
"Type": "AWS::IAM::Role",
|
|
89
|
-
"Properties": {
|
|
90
|
-
"RoleName": "${batchServiceRoleName}",
|
|
91
|
-
"Path": "/",
|
|
92
|
-
"AssumeRolePolicyDocument": {
|
|
93
|
-
"Version": "2008-10-17",
|
|
94
|
-
"Statement": [
|
|
95
|
-
{
|
|
96
|
-
"Sid": "",
|
|
97
|
-
"Effect": "Allow",
|
|
98
|
-
"Principal": {
|
|
99
|
-
"Service": "batch.amazonaws.com"
|
|
100
|
-
},
|
|
101
|
-
"Action": "sts:AssumeRole"
|
|
102
|
-
}
|
|
103
|
-
]
|
|
104
|
-
},
|
|
105
|
-
"ManagedPolicyArns": [
|
|
106
|
-
"arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole"
|
|
107
|
-
]
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
`;
|
|
111
|
-
|
|
112
|
-
return {
|
|
113
|
-
[this.provider.naming.getBatchServiceRoleLogicalId()]: JSON.parse(batchServiceRoleTemplate)
|
|
114
|
-
};
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
/**
|
|
118
|
-
* Generates the IAM Service Role Object that will be used on instances within our compute environment to launch containers
|
|
119
|
-
*/
|
|
120
|
-
function generateBatchInstanceRole() {
|
|
121
|
-
const batchInstanceRoleName = `BatchInstanceRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
122
|
-
const batchInstanceManagementRoleTemplate = `
|
|
123
|
-
{
|
|
124
|
-
"Type": "AWS::IAM::Role",
|
|
125
|
-
"Properties": {
|
|
126
|
-
"RoleName": "${batchInstanceRoleName}",
|
|
127
|
-
"Path": "/",
|
|
128
|
-
"AssumeRolePolicyDocument": {
|
|
129
|
-
"Version": "2008-10-17",
|
|
130
|
-
"Statement": [
|
|
131
|
-
{
|
|
132
|
-
"Sid": "",
|
|
133
|
-
"Effect": "Allow",
|
|
134
|
-
"Principal": {
|
|
135
|
-
"Service": "ec2.amazonaws.com"
|
|
136
|
-
},
|
|
137
|
-
"Action": "sts:AssumeRole"
|
|
138
|
-
}
|
|
139
|
-
]
|
|
140
|
-
},
|
|
141
|
-
"ManagedPolicyArns": [
|
|
142
|
-
"arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
|
|
143
|
-
]
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
`;
|
|
147
|
-
|
|
148
|
-
const batchInstanceProfileTemplate = `
|
|
149
|
-
{
|
|
150
|
-
"Type": "AWS::IAM::InstanceProfile",
|
|
151
|
-
"Properties": {
|
|
152
|
-
"Path": "/",
|
|
153
|
-
"Roles": [
|
|
154
|
-
{
|
|
155
|
-
"Ref": "${this.provider.naming.getBatchInstanceManagementRoleLogicalId()}"
|
|
156
|
-
}
|
|
157
|
-
]
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
`
|
|
161
|
-
|
|
162
|
-
// Setup the JobQueue to push tasks to
|
|
163
|
-
return {
|
|
164
|
-
[this.provider.naming.getBatchInstanceManagementRoleLogicalId()]: JSON.parse(batchInstanceManagementRoleTemplate),
|
|
165
|
-
[this.provider.naming.getBatchInstanceManagementProfileLogicalId()]: JSON.parse(batchInstanceProfileTemplate)
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* Generates the IAM Service Role Object that will be used to manage spot instances in the compute environment
|
|
171
|
-
*/
|
|
172
|
-
function generateBatchSpotFleetRole() {
|
|
173
|
-
const batchSpotRoleName = `BatchSpotFleetRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
174
|
-
const batchSpotRoleManagementTemplate = `
|
|
175
|
-
{
|
|
176
|
-
"Type": "AWS::IAM::Role",
|
|
177
|
-
"Properties": {
|
|
178
|
-
"RoleName": "${batchSpotRoleName}",
|
|
179
|
-
"Path": "/",
|
|
180
|
-
"AssumeRolePolicyDocument": {
|
|
181
|
-
"Version":"2008-10-17",
|
|
182
|
-
"Statement":[
|
|
183
|
-
{
|
|
184
|
-
"Sid":"",
|
|
185
|
-
"Effect":"Allow",
|
|
186
|
-
"Principal":{
|
|
187
|
-
"Service":"spotfleet.amazonaws.com"
|
|
188
|
-
},
|
|
189
|
-
"Action":"sts:AssumeRole"
|
|
190
|
-
}
|
|
191
|
-
]
|
|
192
|
-
},
|
|
193
|
-
"ManagedPolicyArns": [
|
|
194
|
-
"arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole"
|
|
195
|
-
]
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
`;
|
|
199
|
-
|
|
200
|
-
return {
|
|
201
|
-
[this.provider.naming.getBatchSpotFleetManagementRoleLogicalId()]: JSON.parse(batchSpotRoleManagementTemplate)
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* Generates the IAM Role that can be used by our lambda "schedule batch" functions
|
|
208
|
-
*/
|
|
209
|
-
function generateLambdaScheduleExecutionRole() {
|
|
210
|
-
const lambdaScheduleExecutionRoleName = `BatchScheduleRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
211
|
-
const lambdaScheduleExecutionRoleTemplate = `
|
|
212
|
-
{
|
|
213
|
-
"Type": "AWS::IAM::Role",
|
|
214
|
-
"Properties": {
|
|
215
|
-
"RoleName": "${lambdaScheduleExecutionRoleName}",
|
|
216
|
-
"Path": "/",
|
|
217
|
-
"AssumeRolePolicyDocument": {
|
|
218
|
-
"Version": "2008-10-17",
|
|
219
|
-
"Statement": [
|
|
220
|
-
{
|
|
221
|
-
"Sid": "",
|
|
222
|
-
"Effect": "Allow",
|
|
223
|
-
"Principal": {
|
|
224
|
-
"Service": "lambda.amazonaws.com"
|
|
225
|
-
},
|
|
226
|
-
"Action": "sts:AssumeRole"
|
|
227
|
-
}
|
|
228
|
-
]
|
|
229
|
-
},
|
|
230
|
-
"Policies": [
|
|
231
|
-
{
|
|
232
|
-
"PolicyName": "lambda-schedule-execution-policies",
|
|
233
|
-
"PolicyDocument": {
|
|
234
|
-
"Version": "2012-10-17",
|
|
235
|
-
"Statement": [
|
|
236
|
-
{
|
|
237
|
-
"Effect": "Allow",
|
|
238
|
-
"Action": [
|
|
239
|
-
"xray:PutTraceSegments",
|
|
240
|
-
"xray:PutTelemetryRecords"
|
|
241
|
-
],
|
|
242
|
-
"Resource": [
|
|
243
|
-
"*"
|
|
244
|
-
]
|
|
245
|
-
},
|
|
246
|
-
{
|
|
247
|
-
"Effect": "Allow",
|
|
248
|
-
"Action": [
|
|
249
|
-
"ec2:CreateNetworkInterface",
|
|
250
|
-
"ec2:DescribeNetworkInterfaces",
|
|
251
|
-
"ec2:DetachNetworkInterface",
|
|
252
|
-
"ec2:DeleteNetworkInterface"
|
|
253
|
-
],
|
|
254
|
-
"Resource": [
|
|
255
|
-
"*"
|
|
256
|
-
]
|
|
257
|
-
},
|
|
258
|
-
{
|
|
259
|
-
"Effect": "Allow",
|
|
260
|
-
"Action": [
|
|
261
|
-
"batch:SubmitJob"
|
|
262
|
-
],
|
|
263
|
-
"Resource": [
|
|
264
|
-
{
|
|
265
|
-
"Ref": "${this.provider.naming.getBatchJobQueueLogicalId()}"
|
|
266
|
-
},
|
|
267
|
-
"arn:aws:batch:*:*:job-definition/*:*"
|
|
268
|
-
]
|
|
269
|
-
}
|
|
270
|
-
]
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
]
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
`;
|
|
277
|
-
|
|
278
|
-
// Setup the JobQueue to push tasks to
|
|
279
|
-
return {
|
|
280
|
-
[this.provider.naming.getLambdaScheduleExecutionRoleLogicalId()]: JSON.parse(lambdaScheduleExecutionRoleTemplate)
|
|
281
|
-
};
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Generates the JobQueue object that we will submit tasks to
|
|
286
|
-
*/
|
|
287
|
-
function generateBatchJobQueue() {
|
|
288
|
-
const batchJobQueueTemplate = `
|
|
289
|
-
{
|
|
290
|
-
"Type": "AWS::Batch::JobQueue",
|
|
291
|
-
"Properties": {
|
|
292
|
-
"JobQueueName": "${this.provider.naming.getBatchJobQueueName()}",
|
|
293
|
-
"Priority": 1,
|
|
294
|
-
"ComputeEnvironmentOrder": [
|
|
295
|
-
{
|
|
296
|
-
"ComputeEnvironment": { "Ref": "${this.provider.naming.getBatchComputeEnvironmentLogicalId()}" },
|
|
297
|
-
"Order": 1
|
|
298
|
-
}
|
|
299
|
-
]
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
`;
|
|
303
|
-
|
|
304
|
-
return {
|
|
305
|
-
[this.provider.naming.getBatchJobQueueLogicalId()]: JSON.parse(batchJobQueueTemplate)
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
/**
|
|
310
|
-
* Generates the ComputeEnvironment Object that will be used to run tasks
|
|
311
|
-
*/
|
|
312
|
-
function generateBatchComputeEnvironment() {
|
|
313
|
-
// Setup our compute environment
|
|
314
|
-
const batchComputeResourceTemplate = `
|
|
315
|
-
{
|
|
316
|
-
"Type": "EC2",
|
|
317
|
-
"InstanceRole": {
|
|
318
|
-
"Fn::GetAtt": [
|
|
319
|
-
"${this.provider.naming.getBatchInstanceManagementProfileLogicalId()}",
|
|
320
|
-
"Arn"
|
|
321
|
-
]
|
|
322
|
-
},
|
|
323
|
-
"SpotIamFleetRole": {
|
|
324
|
-
"Fn::GetAtt": [
|
|
325
|
-
"${this.provider.naming.getBatchSpotFleetManagementRoleLogicalId()}",
|
|
326
|
-
"Arn"
|
|
327
|
-
]
|
|
328
|
-
},
|
|
329
|
-
"InstanceTypes": [
|
|
330
|
-
"c5.large"
|
|
331
|
-
],
|
|
332
|
-
"MinvCpus": 0,
|
|
333
|
-
"MaxvCpus": 2,
|
|
334
|
-
"Tags": {
|
|
335
|
-
"Name": "AWS Batch Instance - ${this.provider.serverless.service.service}-${this.provider.getStage()}"
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
`;
|
|
339
|
-
|
|
340
|
-
// Merge any overrides into our compute environment template
|
|
341
|
-
const batchComputeResourceObject = _.merge(
|
|
342
|
-
{},
|
|
343
|
-
JSON.parse(batchComputeResourceTemplate),
|
|
344
|
-
this.serverless.service.provider.batch
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
// If we are a SPOT type, default the BigPercentage to 100% (always pay lowest market price)
|
|
348
|
-
if (batchComputeResourceObject.hasOwnProperty("Type")
|
|
349
|
-
&& batchComputeResourceObject.Type == "SPOT"
|
|
350
|
-
&& ! batchComputeResourceObject.hasOwnProperty("BidPercentage")) {
|
|
351
|
-
|
|
352
|
-
batchComputeResourceObject["BidPercentage"] = 100;
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
const computeEnvironmentName = `${this.provider.serverless.service.service}-${this.provider.getStage()}-ComputeEnvironment-${Math.floor(Math.random() * Math.floor(1000000))}`.substring(0, 64);
|
|
356
|
-
const batchComputeEnvironmentTemplate = `
|
|
357
|
-
{
|
|
358
|
-
"Type" : "AWS::Batch::ComputeEnvironment",
|
|
359
|
-
"Properties" : {
|
|
360
|
-
"ComputeEnvironmentName" : "${computeEnvironmentName}",
|
|
361
|
-
"ServiceRole" : {
|
|
362
|
-
"Fn::GetAtt": [
|
|
363
|
-
"${this.provider.naming.getBatchServiceRoleLogicalId()}",
|
|
364
|
-
"Arn"
|
|
365
|
-
]
|
|
366
|
-
},
|
|
367
|
-
"Type" : "MANAGED",
|
|
368
|
-
"ComputeResources": ${JSON.stringify(batchComputeResourceObject)}
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
`;
|
|
372
|
-
|
|
373
|
-
// Then merge the compute resource into the Compute Environment object
|
|
374
|
-
return {
|
|
375
|
-
[this.provider.naming.getBatchComputeEnvironmentLogicalId()]: JSON.parse(batchComputeEnvironmentTemplate)
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
/**
|
|
380
|
-
* Adds the AWS Batch Compute Environment, Job Queue, and Job Definition to our cloud formation
|
|
381
|
-
* template
|
|
382
|
-
*/
|
|
383
|
-
function generateAWSBatchTemplate() {
|
|
384
|
-
this.serverless.cli.log("Generating AWS Batch");
|
|
385
|
-
|
|
386
|
-
const newBatchServiceRoleObject = generateBatchServiceRole.bind(this)();
|
|
387
|
-
const newBatchInstanceManagementRoleObject = generateBatchInstanceRole.bind(this)();
|
|
388
|
-
const newBatchSpotFleetManagementObject = generateBatchSpotFleetRole.bind(this)();
|
|
389
|
-
const newLambdaScheduleExecutionRoleObject = generateLambdaScheduleExecutionRole.bind(this)();
|
|
390
|
-
const newBatchJobQueueObject = generateBatchJobQueue.bind(this)();
|
|
391
|
-
const newBatchComputeEnvironmentObject = generateBatchComputeEnvironment.bind(this)();
|
|
392
|
-
|
|
393
|
-
// Add it to our initial compiled cloud formation templates
|
|
394
|
-
_.merge(
|
|
395
|
-
this.serverless.service.provider.compiledCloudFormationTemplate.Resources,
|
|
396
|
-
newBatchServiceRoleObject,
|
|
397
|
-
newBatchInstanceManagementRoleObject,
|
|
398
|
-
newBatchSpotFleetManagementObject,
|
|
399
|
-
newLambdaScheduleExecutionRoleObject,
|
|
400
|
-
newBatchJobQueueObject,
|
|
401
|
-
newBatchComputeEnvironmentObject,
|
|
402
|
-
);
|
|
403
|
-
|
|
404
|
-
return BbPromise.resolve();
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
module.exports = {
|
|
408
|
-
getBatchServiceRoleLogicalId,
|
|
409
|
-
getBatchInstanceManagementRoleLogicalId,
|
|
410
|
-
getBatchInstanceManagementProfileLogicalId,
|
|
411
|
-
getBatchSpotFleetManagementRoleLogicalId,
|
|
412
|
-
getLambdaScheduleExecutionRoleLogicalId,
|
|
413
|
-
getBatchComputeEnvironmentLogicalId,
|
|
414
|
-
getBatchJobQueueLogicalId,
|
|
415
|
-
getBatchJobQueueName,
|
|
416
|
-
validateAWSBatchServerlessConfig,
|
|
417
|
-
generateAWSBatchTemplate
|
|
418
|
-
};
|
|
1
|
+
const BbPromise = require('bluebird');
|
|
2
|
+
const _ = require('lodash');
|
|
3
|
+
const util = require('util');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @returns {string} "IamRoleBatchService"
|
|
7
|
+
*/
|
|
8
|
+
function getBatchServiceRoleLogicalId() {
|
|
9
|
+
return "IamRoleBatchService";
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @returns {string} "IamRoleBatchInstanceManagement"
|
|
14
|
+
*/
|
|
15
|
+
function getBatchInstanceManagementRoleLogicalId() {
|
|
16
|
+
return "IamRoleBatchInstanceManagement";
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @returns {string} "IamProfileBatchInstanceManagement"
|
|
21
|
+
*/
|
|
22
|
+
function getBatchInstanceManagementProfileLogicalId() {
|
|
23
|
+
return "IamProfileBatchInstanceManagement";
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* @returns {string} "IamRoleBatchSpotFleetManagement"
|
|
28
|
+
*/
|
|
29
|
+
function getBatchSpotFleetManagementRoleLogicalId() {
|
|
30
|
+
return "IamRoleBatchSpotFleetManagement";
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @returns {string} "IamRoleLambdaScheduleExecution"
|
|
35
|
+
*/
|
|
36
|
+
function getLambdaScheduleExecutionRoleLogicalId() {
|
|
37
|
+
return "IamRoleLambdaScheduleExecution";
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @returns {string} "BatchComputeEnvironment"
|
|
42
|
+
*/
|
|
43
|
+
function getBatchComputeEnvironmentLogicalId() {
|
|
44
|
+
return "BatchComputeEnvironment";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @returns {string} "BatchJobQueue"
|
|
49
|
+
*/
|
|
50
|
+
function getBatchJobQueueLogicalId() {
|
|
51
|
+
return "BatchJobQueue";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* @returns {string} The name of the job queue to be used when submitting the job
|
|
56
|
+
*/
|
|
57
|
+
function getBatchJobQueueName() {
|
|
58
|
+
return `${this.provider.serverless.service.service}-${this.provider.getStage()}-JobQueue`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Validates the "batch" object in the serverless config to ensure that we have:
|
|
63
|
+
* - subnets
|
|
64
|
+
* - securityGroups
|
|
65
|
+
*/
|
|
66
|
+
function validateAWSBatchServerlessConfig() {
|
|
67
|
+
const provider = this.serverless.service.provider;
|
|
68
|
+
if (! provider.hasOwnProperty("batch")) {
|
|
69
|
+
throw new Error("'batch' configuration not defined on the provider");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const batch = provider.batch;
|
|
73
|
+
if (! batch.hasOwnProperty("SecurityGroupIds")) {
|
|
74
|
+
throw new Error("'batch' configuration does not contain property 'SecurityGroupIds' (make sure it's capitalized)");
|
|
75
|
+
}
|
|
76
|
+
if (! batch.hasOwnProperty("Subnets")) {
|
|
77
|
+
throw new Error("'batch' configuration does not contain property 'Subnets' (make sure it's capitalized)");
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Generates the IAM Service Role Object to be used by the Batch Compute Environment
|
|
83
|
+
*/
|
|
84
|
+
function generateBatchServiceRole() {
|
|
85
|
+
const batchServiceRoleName = `BatchServiceRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
86
|
+
const batchServiceRoleTemplate = `
|
|
87
|
+
{
|
|
88
|
+
"Type": "AWS::IAM::Role",
|
|
89
|
+
"Properties": {
|
|
90
|
+
"RoleName": "${batchServiceRoleName}",
|
|
91
|
+
"Path": "/",
|
|
92
|
+
"AssumeRolePolicyDocument": {
|
|
93
|
+
"Version": "2008-10-17",
|
|
94
|
+
"Statement": [
|
|
95
|
+
{
|
|
96
|
+
"Sid": "",
|
|
97
|
+
"Effect": "Allow",
|
|
98
|
+
"Principal": {
|
|
99
|
+
"Service": "batch.amazonaws.com"
|
|
100
|
+
},
|
|
101
|
+
"Action": "sts:AssumeRole"
|
|
102
|
+
}
|
|
103
|
+
]
|
|
104
|
+
},
|
|
105
|
+
"ManagedPolicyArns": [
|
|
106
|
+
"arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole"
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
`;
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
[this.provider.naming.getBatchServiceRoleLogicalId()]: JSON.parse(batchServiceRoleTemplate)
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Generates the IAM Service Role Object that will be used on instances within our compute environment to launch containers
|
|
119
|
+
*/
|
|
120
|
+
function generateBatchInstanceRole() {
|
|
121
|
+
const batchInstanceRoleName = `BatchInstanceRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
122
|
+
const batchInstanceManagementRoleTemplate = `
|
|
123
|
+
{
|
|
124
|
+
"Type": "AWS::IAM::Role",
|
|
125
|
+
"Properties": {
|
|
126
|
+
"RoleName": "${batchInstanceRoleName}",
|
|
127
|
+
"Path": "/",
|
|
128
|
+
"AssumeRolePolicyDocument": {
|
|
129
|
+
"Version": "2008-10-17",
|
|
130
|
+
"Statement": [
|
|
131
|
+
{
|
|
132
|
+
"Sid": "",
|
|
133
|
+
"Effect": "Allow",
|
|
134
|
+
"Principal": {
|
|
135
|
+
"Service": "ec2.amazonaws.com"
|
|
136
|
+
},
|
|
137
|
+
"Action": "sts:AssumeRole"
|
|
138
|
+
}
|
|
139
|
+
]
|
|
140
|
+
},
|
|
141
|
+
"ManagedPolicyArns": [
|
|
142
|
+
"arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
|
|
143
|
+
]
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
`;
|
|
147
|
+
|
|
148
|
+
const batchInstanceProfileTemplate = `
|
|
149
|
+
{
|
|
150
|
+
"Type": "AWS::IAM::InstanceProfile",
|
|
151
|
+
"Properties": {
|
|
152
|
+
"Path": "/",
|
|
153
|
+
"Roles": [
|
|
154
|
+
{
|
|
155
|
+
"Ref": "${this.provider.naming.getBatchInstanceManagementRoleLogicalId()}"
|
|
156
|
+
}
|
|
157
|
+
]
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
`
|
|
161
|
+
|
|
162
|
+
// Setup the JobQueue to push tasks to
|
|
163
|
+
return {
|
|
164
|
+
[this.provider.naming.getBatchInstanceManagementRoleLogicalId()]: JSON.parse(batchInstanceManagementRoleTemplate),
|
|
165
|
+
[this.provider.naming.getBatchInstanceManagementProfileLogicalId()]: JSON.parse(batchInstanceProfileTemplate)
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Generates the IAM Service Role Object that will be used to manage spot instances in the compute environment
|
|
171
|
+
*/
|
|
172
|
+
function generateBatchSpotFleetRole() {
|
|
173
|
+
const batchSpotRoleName = `BatchSpotFleetRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
174
|
+
const batchSpotRoleManagementTemplate = `
|
|
175
|
+
{
|
|
176
|
+
"Type": "AWS::IAM::Role",
|
|
177
|
+
"Properties": {
|
|
178
|
+
"RoleName": "${batchSpotRoleName}",
|
|
179
|
+
"Path": "/",
|
|
180
|
+
"AssumeRolePolicyDocument": {
|
|
181
|
+
"Version":"2008-10-17",
|
|
182
|
+
"Statement":[
|
|
183
|
+
{
|
|
184
|
+
"Sid":"",
|
|
185
|
+
"Effect":"Allow",
|
|
186
|
+
"Principal":{
|
|
187
|
+
"Service":"spotfleet.amazonaws.com"
|
|
188
|
+
},
|
|
189
|
+
"Action":"sts:AssumeRole"
|
|
190
|
+
}
|
|
191
|
+
]
|
|
192
|
+
},
|
|
193
|
+
"ManagedPolicyArns": [
|
|
194
|
+
"arn:aws:iam::aws:policy/service-role/AmazonEC2SpotFleetTaggingRole"
|
|
195
|
+
]
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
`;
|
|
199
|
+
|
|
200
|
+
return {
|
|
201
|
+
[this.provider.naming.getBatchSpotFleetManagementRoleLogicalId()]: JSON.parse(batchSpotRoleManagementTemplate)
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Generates the IAM Role that can be used by our lambda "schedule batch" functions
|
|
208
|
+
*/
|
|
209
|
+
function generateLambdaScheduleExecutionRole() {
|
|
210
|
+
const lambdaScheduleExecutionRoleName = `BatchScheduleRole-${this.provider.getRegion()}-${this.provider.getStage()}-${this.provider.serverless.service.service}`.substring(0, 64);
|
|
211
|
+
const lambdaScheduleExecutionRoleTemplate = `
|
|
212
|
+
{
|
|
213
|
+
"Type": "AWS::IAM::Role",
|
|
214
|
+
"Properties": {
|
|
215
|
+
"RoleName": "${lambdaScheduleExecutionRoleName}",
|
|
216
|
+
"Path": "/",
|
|
217
|
+
"AssumeRolePolicyDocument": {
|
|
218
|
+
"Version": "2008-10-17",
|
|
219
|
+
"Statement": [
|
|
220
|
+
{
|
|
221
|
+
"Sid": "",
|
|
222
|
+
"Effect": "Allow",
|
|
223
|
+
"Principal": {
|
|
224
|
+
"Service": "lambda.amazonaws.com"
|
|
225
|
+
},
|
|
226
|
+
"Action": "sts:AssumeRole"
|
|
227
|
+
}
|
|
228
|
+
]
|
|
229
|
+
},
|
|
230
|
+
"Policies": [
|
|
231
|
+
{
|
|
232
|
+
"PolicyName": "lambda-schedule-execution-policies",
|
|
233
|
+
"PolicyDocument": {
|
|
234
|
+
"Version": "2012-10-17",
|
|
235
|
+
"Statement": [
|
|
236
|
+
{
|
|
237
|
+
"Effect": "Allow",
|
|
238
|
+
"Action": [
|
|
239
|
+
"xray:PutTraceSegments",
|
|
240
|
+
"xray:PutTelemetryRecords"
|
|
241
|
+
],
|
|
242
|
+
"Resource": [
|
|
243
|
+
"*"
|
|
244
|
+
]
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
"Effect": "Allow",
|
|
248
|
+
"Action": [
|
|
249
|
+
"ec2:CreateNetworkInterface",
|
|
250
|
+
"ec2:DescribeNetworkInterfaces",
|
|
251
|
+
"ec2:DetachNetworkInterface",
|
|
252
|
+
"ec2:DeleteNetworkInterface"
|
|
253
|
+
],
|
|
254
|
+
"Resource": [
|
|
255
|
+
"*"
|
|
256
|
+
]
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"Effect": "Allow",
|
|
260
|
+
"Action": [
|
|
261
|
+
"batch:SubmitJob"
|
|
262
|
+
],
|
|
263
|
+
"Resource": [
|
|
264
|
+
{
|
|
265
|
+
"Ref": "${this.provider.naming.getBatchJobQueueLogicalId()}"
|
|
266
|
+
},
|
|
267
|
+
"arn:aws:batch:*:*:job-definition/*:*"
|
|
268
|
+
]
|
|
269
|
+
}
|
|
270
|
+
]
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
]
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
`;
|
|
277
|
+
|
|
278
|
+
// Setup the JobQueue to push tasks to
|
|
279
|
+
return {
|
|
280
|
+
[this.provider.naming.getLambdaScheduleExecutionRoleLogicalId()]: JSON.parse(lambdaScheduleExecutionRoleTemplate)
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Generates the JobQueue object that we will submit tasks to
|
|
286
|
+
*/
|
|
287
|
+
function generateBatchJobQueue() {
|
|
288
|
+
const batchJobQueueTemplate = `
|
|
289
|
+
{
|
|
290
|
+
"Type": "AWS::Batch::JobQueue",
|
|
291
|
+
"Properties": {
|
|
292
|
+
"JobQueueName": "${this.provider.naming.getBatchJobQueueName()}",
|
|
293
|
+
"Priority": 1,
|
|
294
|
+
"ComputeEnvironmentOrder": [
|
|
295
|
+
{
|
|
296
|
+
"ComputeEnvironment": { "Ref": "${this.provider.naming.getBatchComputeEnvironmentLogicalId()}" },
|
|
297
|
+
"Order": 1
|
|
298
|
+
}
|
|
299
|
+
]
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
`;
|
|
303
|
+
|
|
304
|
+
return {
|
|
305
|
+
[this.provider.naming.getBatchJobQueueLogicalId()]: JSON.parse(batchJobQueueTemplate)
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Generates the ComputeEnvironment Object that will be used to run tasks
|
|
311
|
+
*/
|
|
312
|
+
function generateBatchComputeEnvironment() {
|
|
313
|
+
// Setup our compute environment
|
|
314
|
+
const batchComputeResourceTemplate = `
|
|
315
|
+
{
|
|
316
|
+
"Type": "EC2",
|
|
317
|
+
"InstanceRole": {
|
|
318
|
+
"Fn::GetAtt": [
|
|
319
|
+
"${this.provider.naming.getBatchInstanceManagementProfileLogicalId()}",
|
|
320
|
+
"Arn"
|
|
321
|
+
]
|
|
322
|
+
},
|
|
323
|
+
"SpotIamFleetRole": {
|
|
324
|
+
"Fn::GetAtt": [
|
|
325
|
+
"${this.provider.naming.getBatchSpotFleetManagementRoleLogicalId()}",
|
|
326
|
+
"Arn"
|
|
327
|
+
]
|
|
328
|
+
},
|
|
329
|
+
"InstanceTypes": [
|
|
330
|
+
"c5.large"
|
|
331
|
+
],
|
|
332
|
+
"MinvCpus": 0,
|
|
333
|
+
"MaxvCpus": 2,
|
|
334
|
+
"Tags": {
|
|
335
|
+
"Name": "AWS Batch Instance - ${this.provider.serverless.service.service}-${this.provider.getStage()}"
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
`;
|
|
339
|
+
|
|
340
|
+
// Merge any overrides into our compute environment template
|
|
341
|
+
const batchComputeResourceObject = _.merge(
|
|
342
|
+
{},
|
|
343
|
+
JSON.parse(batchComputeResourceTemplate),
|
|
344
|
+
this.serverless.service.provider.batch
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
// If we are a SPOT type, default the BigPercentage to 100% (always pay lowest market price)
|
|
348
|
+
if (batchComputeResourceObject.hasOwnProperty("Type")
|
|
349
|
+
&& batchComputeResourceObject.Type == "SPOT"
|
|
350
|
+
&& ! batchComputeResourceObject.hasOwnProperty("BidPercentage")) {
|
|
351
|
+
|
|
352
|
+
batchComputeResourceObject["BidPercentage"] = 100;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const computeEnvironmentName = `${this.provider.serverless.service.service}-${this.provider.getStage()}-ComputeEnvironment-${Math.floor(Math.random() * Math.floor(1000000))}`.substring(0, 64);
|
|
356
|
+
const batchComputeEnvironmentTemplate = `
|
|
357
|
+
{
|
|
358
|
+
"Type" : "AWS::Batch::ComputeEnvironment",
|
|
359
|
+
"Properties" : {
|
|
360
|
+
"ComputeEnvironmentName" : "${computeEnvironmentName}",
|
|
361
|
+
"ServiceRole" : {
|
|
362
|
+
"Fn::GetAtt": [
|
|
363
|
+
"${this.provider.naming.getBatchServiceRoleLogicalId()}",
|
|
364
|
+
"Arn"
|
|
365
|
+
]
|
|
366
|
+
},
|
|
367
|
+
"Type" : "MANAGED",
|
|
368
|
+
"ComputeResources": ${JSON.stringify(batchComputeResourceObject)}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
`;
|
|
372
|
+
|
|
373
|
+
// Then merge the compute resource into the Compute Environment object
|
|
374
|
+
return {
|
|
375
|
+
[this.provider.naming.getBatchComputeEnvironmentLogicalId()]: JSON.parse(batchComputeEnvironmentTemplate)
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Adds the AWS Batch Compute Environment, Job Queue, and Job Definition to our cloud formation
|
|
381
|
+
* template
|
|
382
|
+
*/
|
|
383
|
+
function generateAWSBatchTemplate() {
|
|
384
|
+
this.serverless.cli.log("Generating AWS Batch");
|
|
385
|
+
|
|
386
|
+
const newBatchServiceRoleObject = generateBatchServiceRole.bind(this)();
|
|
387
|
+
const newBatchInstanceManagementRoleObject = generateBatchInstanceRole.bind(this)();
|
|
388
|
+
const newBatchSpotFleetManagementObject = generateBatchSpotFleetRole.bind(this)();
|
|
389
|
+
const newLambdaScheduleExecutionRoleObject = generateLambdaScheduleExecutionRole.bind(this)();
|
|
390
|
+
const newBatchJobQueueObject = generateBatchJobQueue.bind(this)();
|
|
391
|
+
const newBatchComputeEnvironmentObject = generateBatchComputeEnvironment.bind(this)();
|
|
392
|
+
|
|
393
|
+
// Add it to our initial compiled cloud formation templates
|
|
394
|
+
_.merge(
|
|
395
|
+
this.serverless.service.provider.compiledCloudFormationTemplate.Resources,
|
|
396
|
+
newBatchServiceRoleObject,
|
|
397
|
+
newBatchInstanceManagementRoleObject,
|
|
398
|
+
newBatchSpotFleetManagementObject,
|
|
399
|
+
newLambdaScheduleExecutionRoleObject,
|
|
400
|
+
newBatchJobQueueObject,
|
|
401
|
+
newBatchComputeEnvironmentObject,
|
|
402
|
+
);
|
|
403
|
+
|
|
404
|
+
return BbPromise.resolve();
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
module.exports = {
|
|
408
|
+
getBatchServiceRoleLogicalId,
|
|
409
|
+
getBatchInstanceManagementRoleLogicalId,
|
|
410
|
+
getBatchInstanceManagementProfileLogicalId,
|
|
411
|
+
getBatchSpotFleetManagementRoleLogicalId,
|
|
412
|
+
getLambdaScheduleExecutionRoleLogicalId,
|
|
413
|
+
getBatchComputeEnvironmentLogicalId,
|
|
414
|
+
getBatchJobQueueLogicalId,
|
|
415
|
+
getBatchJobQueueName,
|
|
416
|
+
validateAWSBatchServerlessConfig,
|
|
417
|
+
generateAWSBatchTemplate
|
|
418
|
+
};
|