@rlyle1179/serverless-aws-batch 1.1.19 → 1.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +362 -362
- package/index.js +66 -66
- package/lib/awscli.js +107 -107
- package/lib/batchenvironment.js +418 -418
- package/lib/batchtask.js +390 -390
- package/lib/docker.js +139 -139
- package/lib/ecr.js +29 -29
- package/lib/generateCoreTemplate.js +120 -120
- package/lib/schedule.js +99 -90
- package/package.json +35 -35
package/index.js
CHANGED
|
@@ -1,66 +1,66 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const BbPromise = require('bluebird');
|
|
4
|
-
const fse = require('fs-extra');
|
|
5
|
-
const generateCoreTemplate = require('./lib/generateCoreTemplate');
|
|
6
|
-
const ecr = require('./lib/ecr');
|
|
7
|
-
const docker = require('./lib/docker');
|
|
8
|
-
const batchenvironment = require('./lib/batchenvironment');
|
|
9
|
-
const batchtask = require('./lib/batchtask');
|
|
10
|
-
const awscli = require('./lib/awscli');
|
|
11
|
-
const _ = require('lodash');
|
|
12
|
-
|
|
13
|
-
BbPromise.promisifyAll(fse);
|
|
14
|
-
|
|
15
|
-
class ServerlessAWSBatch {
|
|
16
|
-
constructor(serverless, options) {
|
|
17
|
-
this.serverless = serverless;
|
|
18
|
-
this.options = options;
|
|
19
|
-
this.provider = this.serverless.getProvider('aws');
|
|
20
|
-
|
|
21
|
-
// Make sure that we add the names for our ECR, docker, and batch resources to the provider
|
|
22
|
-
_.merge(
|
|
23
|
-
this.provider.naming,
|
|
24
|
-
{
|
|
25
|
-
'getECRLogicalId': ecr.getECRLogicalId,
|
|
26
|
-
'getECRRepositoryName': ecr.getECRRepositoryName,
|
|
27
|
-
'getECRRepositoryURL': ecr.getECRRepositoryURL,
|
|
28
|
-
'getDockerImageName': docker.getDockerImageName,
|
|
29
|
-
'getBatchServiceRoleLogicalId': batchenvironment.getBatchServiceRoleLogicalId,
|
|
30
|
-
'getBatchInstanceManagementRoleLogicalId': batchenvironment.getBatchInstanceManagementRoleLogicalId,
|
|
31
|
-
'getBatchInstanceManagementProfileLogicalId': batchenvironment.getBatchInstanceManagementProfileLogicalId,
|
|
32
|
-
'getBatchSpotFleetManagementRoleLogicalId': batchenvironment.getBatchSpotFleetManagementRoleLogicalId,
|
|
33
|
-
'getBatchJobExecutionRoleLogicalId': batchtask.getBatchJobExecutionRoleLogicalId,
|
|
34
|
-
'getLambdaScheduleExecutionRoleLogicalId': batchenvironment.getLambdaScheduleExecutionRoleLogicalId,
|
|
35
|
-
'getBatchComputeEnvironmentLogicalId': batchenvironment.getBatchComputeEnvironmentLogicalId,
|
|
36
|
-
'getBatchJobQueueLogicalId': batchenvironment.getBatchJobQueueLogicalId,
|
|
37
|
-
'getBatchJobQueueName': batchenvironment.getBatchJobQueueName,
|
|
38
|
-
'getJobDefinitionLogicalId': batchtask.getJobDefinitionLogicalId
|
|
39
|
-
}
|
|
40
|
-
);
|
|
41
|
-
|
|
42
|
-
// Define inner lifecycles
|
|
43
|
-
this.commands = {}
|
|
44
|
-
|
|
45
|
-
this.hooks = {
|
|
46
|
-
'after:package:initialize': () => BbPromise.bind(this)
|
|
47
|
-
.then(generateCoreTemplate.generateCoreTemplate),
|
|
48
|
-
|
|
49
|
-
'before:package:compileFunctions': () => BbPromise.bind(this)
|
|
50
|
-
.then(batchenvironment.validateAWSBatchServerlessConfig)
|
|
51
|
-
.then(batchenvironment.generateAWSBatchTemplate)
|
|
52
|
-
.then(batchtask.compileBatchTasks),
|
|
53
|
-
|
|
54
|
-
'after:package:createDeploymentArtifacts': () => BbPromise.bind(this)
|
|
55
|
-
.then(docker.buildDockerImage),
|
|
56
|
-
|
|
57
|
-
'after:aws:deploy:deploy:updateStack': () => BbPromise.bind(this)
|
|
58
|
-
.then(docker.pushDockerImageToECR),
|
|
59
|
-
|
|
60
|
-
'before:remove:remove': () => BbPromise.bind(this)
|
|
61
|
-
.then(awscli.deleteAllDockerImagesInECR)
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
module.exports = ServerlessAWSBatch;
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const BbPromise = require('bluebird');
|
|
4
|
+
const fse = require('fs-extra');
|
|
5
|
+
const generateCoreTemplate = require('./lib/generateCoreTemplate');
|
|
6
|
+
const ecr = require('./lib/ecr');
|
|
7
|
+
const docker = require('./lib/docker');
|
|
8
|
+
const batchenvironment = require('./lib/batchenvironment');
|
|
9
|
+
const batchtask = require('./lib/batchtask');
|
|
10
|
+
const awscli = require('./lib/awscli');
|
|
11
|
+
const _ = require('lodash');
|
|
12
|
+
|
|
13
|
+
BbPromise.promisifyAll(fse);
|
|
14
|
+
|
|
15
|
+
class ServerlessAWSBatch {
|
|
16
|
+
constructor(serverless, options) {
|
|
17
|
+
this.serverless = serverless;
|
|
18
|
+
this.options = options;
|
|
19
|
+
this.provider = this.serverless.getProvider('aws');
|
|
20
|
+
|
|
21
|
+
// Make sure that we add the names for our ECR, docker, and batch resources to the provider
|
|
22
|
+
_.merge(
|
|
23
|
+
this.provider.naming,
|
|
24
|
+
{
|
|
25
|
+
'getECRLogicalId': ecr.getECRLogicalId,
|
|
26
|
+
'getECRRepositoryName': ecr.getECRRepositoryName,
|
|
27
|
+
'getECRRepositoryURL': ecr.getECRRepositoryURL,
|
|
28
|
+
'getDockerImageName': docker.getDockerImageName,
|
|
29
|
+
'getBatchServiceRoleLogicalId': batchenvironment.getBatchServiceRoleLogicalId,
|
|
30
|
+
'getBatchInstanceManagementRoleLogicalId': batchenvironment.getBatchInstanceManagementRoleLogicalId,
|
|
31
|
+
'getBatchInstanceManagementProfileLogicalId': batchenvironment.getBatchInstanceManagementProfileLogicalId,
|
|
32
|
+
'getBatchSpotFleetManagementRoleLogicalId': batchenvironment.getBatchSpotFleetManagementRoleLogicalId,
|
|
33
|
+
'getBatchJobExecutionRoleLogicalId': batchtask.getBatchJobExecutionRoleLogicalId,
|
|
34
|
+
'getLambdaScheduleExecutionRoleLogicalId': batchenvironment.getLambdaScheduleExecutionRoleLogicalId,
|
|
35
|
+
'getBatchComputeEnvironmentLogicalId': batchenvironment.getBatchComputeEnvironmentLogicalId,
|
|
36
|
+
'getBatchJobQueueLogicalId': batchenvironment.getBatchJobQueueLogicalId,
|
|
37
|
+
'getBatchJobQueueName': batchenvironment.getBatchJobQueueName,
|
|
38
|
+
'getJobDefinitionLogicalId': batchtask.getJobDefinitionLogicalId
|
|
39
|
+
}
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
// Define inner lifecycles
|
|
43
|
+
this.commands = {}
|
|
44
|
+
|
|
45
|
+
this.hooks = {
|
|
46
|
+
'after:package:initialize': () => BbPromise.bind(this)
|
|
47
|
+
.then(generateCoreTemplate.generateCoreTemplate),
|
|
48
|
+
|
|
49
|
+
'before:package:compileFunctions': () => BbPromise.bind(this)
|
|
50
|
+
.then(batchenvironment.validateAWSBatchServerlessConfig)
|
|
51
|
+
.then(batchenvironment.generateAWSBatchTemplate)
|
|
52
|
+
.then(batchtask.compileBatchTasks),
|
|
53
|
+
|
|
54
|
+
'after:package:createDeploymentArtifacts': () => BbPromise.bind(this)
|
|
55
|
+
.then(docker.buildDockerImage),
|
|
56
|
+
|
|
57
|
+
'after:aws:deploy:deploy:updateStack': () => BbPromise.bind(this)
|
|
58
|
+
.then(docker.pushDockerImageToECR),
|
|
59
|
+
|
|
60
|
+
'before:remove:remove': () => BbPromise.bind(this)
|
|
61
|
+
.then(awscli.deleteAllDockerImagesInECR)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
module.exports = ServerlessAWSBatch;
|
package/lib/awscli.js
CHANGED
|
@@ -1,107 +1,107 @@
|
|
|
1
|
-
const { spawnSync } = require('child_process');
|
|
2
|
-
const fse = require('fs-extra');
|
|
3
|
-
const path = require('path');
|
|
4
|
-
const util = require('util');
|
|
5
|
-
const _ = require('lodash');
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* Helper function to run an AWS CLI command
|
|
9
|
-
* @param {string[]} options
|
|
10
|
-
* @return {Object}
|
|
11
|
-
*/
|
|
12
|
-
function awsCommand(options) {
|
|
13
|
-
const cmd = 'aws';
|
|
14
|
-
const ps = spawnSync(cmd, options, { encoding: 'utf-8' });
|
|
15
|
-
if (ps.error) {
|
|
16
|
-
if (ps.error.code === 'ENOENT') {
|
|
17
|
-
throw new Error('aws cli not found! Please install it.');
|
|
18
|
-
}
|
|
19
|
-
throw new Error(ps.error);
|
|
20
|
-
} else if (ps.status !== 0) {
|
|
21
|
-
throw new Error(ps.stderr);
|
|
22
|
-
}
|
|
23
|
-
return ps;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* Uses the AWS CLI to generate a pre-authenticated docker command that can be
|
|
28
|
-
* used to login to the Elastic Container Registry (ECR) from docker.
|
|
29
|
-
*
|
|
30
|
-
* @return {string} login -u AWS -p <password> https://<account_id>.dkr.ecr.<region>.amazonaws.com
|
|
31
|
-
*/
|
|
32
|
-
function getDockerLoginToECRCommand() {
|
|
33
|
-
const awsCliVersion = awsCommand(['--version']);
|
|
34
|
-
|
|
35
|
-
// AWS CLI v2+ doesn't support "get-login" command any more - https://github.com/aws/aws-cli/issues/2875
|
|
36
|
-
if (awsCliVersion.stdout.trim().indexOf('aws-cli/2') >= 0) {
|
|
37
|
-
const passwordResult = awsCommand(['ecr', 'get-login-password', '--region', this.options.region]);
|
|
38
|
-
// Getting account ID
|
|
39
|
-
const accountIdResult = awsCommand([
|
|
40
|
-
'sts', 'get-caller-identity',
|
|
41
|
-
'--region', this.options.region,
|
|
42
|
-
'--query', 'Account',
|
|
43
|
-
'--output', 'text',
|
|
44
|
-
...(this.options.awsProfile ? ['--profile', this.options.awsProfile] : []),
|
|
45
|
-
]);
|
|
46
|
-
|
|
47
|
-
return 'login -u AWS -p ' + passwordResult.stdout + ' ' +
|
|
48
|
-
accountIdResult.stdout.trim() + '.dkr.ecr.' + this.options.region + '.amazonaws.com';
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
const result = awsCommand(['ecr', 'get-login', '--region', this.options.region, '--no-include-email']);
|
|
52
|
-
// AWS CLI returns the full command with "docker " out front. Remove it since we don't need it.
|
|
53
|
-
return result.stdout.replace('docker ', '').replace(/[\n\r]/g, '');
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Deletes all of our docker images in the remote ECR registry
|
|
58
|
-
*/
|
|
59
|
-
function deleteAllDockerImagesInECR() {
|
|
60
|
-
try {
|
|
61
|
-
const result = awsCommand(['ecr', 'list-images', '--repository-name', this.provider.naming.getECRRepositoryName()]);
|
|
62
|
-
const images = JSON.parse(result.stdout);
|
|
63
|
-
|
|
64
|
-
this.serverless.cli.log(`Found ${images['imageIds'].length} existing ECR images. Deleting...`)
|
|
65
|
-
|
|
66
|
-
// Delete individually instead of batch so we don't run into any CLI length limits
|
|
67
|
-
_.forEach(
|
|
68
|
-
images['imageIds'],
|
|
69
|
-
image => {
|
|
70
|
-
awsCommand([
|
|
71
|
-
'ecr', 'batch-delete-image',
|
|
72
|
-
'--repository-name', this.provider.naming.getECRRepositoryName(),
|
|
73
|
-
'--image-ids', `imageDigest=${image['imageDigest']}`
|
|
74
|
-
]);
|
|
75
|
-
}
|
|
76
|
-
);
|
|
77
|
-
}
|
|
78
|
-
catch (e) {
|
|
79
|
-
// If we failed to delete the stack from a previous "sls-remove" attempt, but already deleted the repository
|
|
80
|
-
if (e.message.indexOf("RepositoryNotFoundException") > -1) {
|
|
81
|
-
this.serverless.cli.log("ECR Repository already deleted");
|
|
82
|
-
}
|
|
83
|
-
// Unknown error
|
|
84
|
-
else {
|
|
85
|
-
throw e;
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Calls AWS CLI to get the accountID of the calling user
|
|
92
|
-
* @returns {string} The accountID of the calling user
|
|
93
|
-
*/
|
|
94
|
-
let awsAccountID = null;
|
|
95
|
-
function getAWSAccountID() {
|
|
96
|
-
if (awsAccountID == null) {
|
|
97
|
-
const result = awsCommand(['sts', 'get-caller-identity']);
|
|
98
|
-
awsAccountID = JSON.parse(result.stdout).Account
|
|
99
|
-
}
|
|
100
|
-
return awsAccountID;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
module.exports = {
|
|
104
|
-
getDockerLoginToECRCommand,
|
|
105
|
-
getAWSAccountID,
|
|
106
|
-
deleteAllDockerImagesInECR
|
|
107
|
-
};
|
|
1
|
+
const { spawnSync } = require('child_process');
|
|
2
|
+
const fse = require('fs-extra');
|
|
3
|
+
const path = require('path');
|
|
4
|
+
const util = require('util');
|
|
5
|
+
const _ = require('lodash');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Helper function to run an AWS CLI command
|
|
9
|
+
* @param {string[]} options
|
|
10
|
+
* @return {Object}
|
|
11
|
+
*/
|
|
12
|
+
function awsCommand(options) {
|
|
13
|
+
const cmd = 'aws';
|
|
14
|
+
const ps = spawnSync(cmd, options, { encoding: 'utf-8' });
|
|
15
|
+
if (ps.error) {
|
|
16
|
+
if (ps.error.code === 'ENOENT') {
|
|
17
|
+
throw new Error('aws cli not found! Please install it.');
|
|
18
|
+
}
|
|
19
|
+
throw new Error(ps.error);
|
|
20
|
+
} else if (ps.status !== 0) {
|
|
21
|
+
throw new Error(ps.stderr);
|
|
22
|
+
}
|
|
23
|
+
return ps;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Uses the AWS CLI to generate a pre-authenticated docker command that can be
|
|
28
|
+
* used to login to the Elastic Container Registry (ECR) from docker.
|
|
29
|
+
*
|
|
30
|
+
* @return {string} login -u AWS -p <password> https://<account_id>.dkr.ecr.<region>.amazonaws.com
|
|
31
|
+
*/
|
|
32
|
+
function getDockerLoginToECRCommand() {
|
|
33
|
+
const awsCliVersion = awsCommand(['--version']);
|
|
34
|
+
|
|
35
|
+
// AWS CLI v2+ doesn't support "get-login" command any more - https://github.com/aws/aws-cli/issues/2875
|
|
36
|
+
if (awsCliVersion.stdout.trim().indexOf('aws-cli/2') >= 0) {
|
|
37
|
+
const passwordResult = awsCommand(['ecr', 'get-login-password', '--region', this.options.region]);
|
|
38
|
+
// Getting account ID
|
|
39
|
+
const accountIdResult = awsCommand([
|
|
40
|
+
'sts', 'get-caller-identity',
|
|
41
|
+
'--region', this.options.region,
|
|
42
|
+
'--query', 'Account',
|
|
43
|
+
'--output', 'text',
|
|
44
|
+
...(this.options.awsProfile ? ['--profile', this.options.awsProfile] : []),
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
return 'login -u AWS -p ' + passwordResult.stdout + ' ' +
|
|
48
|
+
accountIdResult.stdout.trim() + '.dkr.ecr.' + this.options.region + '.amazonaws.com';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const result = awsCommand(['ecr', 'get-login', '--region', this.options.region, '--no-include-email']);
|
|
52
|
+
// AWS CLI returns the full command with "docker " out front. Remove it since we don't need it.
|
|
53
|
+
return result.stdout.replace('docker ', '').replace(/[\n\r]/g, '');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Deletes all of our docker images in the remote ECR registry
|
|
58
|
+
*/
|
|
59
|
+
function deleteAllDockerImagesInECR() {
|
|
60
|
+
try {
|
|
61
|
+
const result = awsCommand(['ecr', 'list-images', '--repository-name', this.provider.naming.getECRRepositoryName()]);
|
|
62
|
+
const images = JSON.parse(result.stdout);
|
|
63
|
+
|
|
64
|
+
this.serverless.cli.log(`Found ${images['imageIds'].length} existing ECR images. Deleting...`)
|
|
65
|
+
|
|
66
|
+
// Delete individually instead of batch so we don't run into any CLI length limits
|
|
67
|
+
_.forEach(
|
|
68
|
+
images['imageIds'],
|
|
69
|
+
image => {
|
|
70
|
+
awsCommand([
|
|
71
|
+
'ecr', 'batch-delete-image',
|
|
72
|
+
'--repository-name', this.provider.naming.getECRRepositoryName(),
|
|
73
|
+
'--image-ids', `imageDigest=${image['imageDigest']}`
|
|
74
|
+
]);
|
|
75
|
+
}
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
catch (e) {
|
|
79
|
+
// If we failed to delete the stack from a previous "sls-remove" attempt, but already deleted the repository
|
|
80
|
+
if (e.message.indexOf("RepositoryNotFoundException") > -1) {
|
|
81
|
+
this.serverless.cli.log("ECR Repository already deleted");
|
|
82
|
+
}
|
|
83
|
+
// Unknown error
|
|
84
|
+
else {
|
|
85
|
+
throw e;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Calls AWS CLI to get the accountID of the calling user
|
|
92
|
+
* @returns {string} The accountID of the calling user
|
|
93
|
+
*/
|
|
94
|
+
let awsAccountID = null;
|
|
95
|
+
function getAWSAccountID() {
|
|
96
|
+
if (awsAccountID == null) {
|
|
97
|
+
const result = awsCommand(['sts', 'get-caller-identity']);
|
|
98
|
+
awsAccountID = JSON.parse(result.stdout).Account
|
|
99
|
+
}
|
|
100
|
+
return awsAccountID;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
module.exports = {
|
|
104
|
+
getDockerLoginToECRCommand,
|
|
105
|
+
getAWSAccountID,
|
|
106
|
+
deleteAllDockerImagesInECR
|
|
107
|
+
};
|