@cumulus/async-operations 11.1.1 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/async_operations.d.ts +2 -0
- package/dist/async_operations.d.ts.map +1 -1
- package/dist/async_operations.js +45 -12
- package/dist/async_operations.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +9 -9
- package/src/async_operations.ts +45 -11
- package/tests/test-async_operations.js +79 -5
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
2
3
|
import { ECS, Lambda } from 'aws-sdk';
|
|
3
4
|
import { EnvironmentVariables } from 'aws-sdk/clients/lambda';
|
|
4
5
|
import { ApiAsyncOperation, AsyncOperationType } from '@cumulus/types/api/async_operations';
|
|
@@ -75,6 +76,7 @@ export declare const createAsyncOperation: (params: {
|
|
|
75
76
|
* @memberof AsyncOperation
|
|
76
77
|
*/
|
|
77
78
|
export declare const startAsyncOperation: (params: {
|
|
79
|
+
asyncOperationId?: string;
|
|
78
80
|
asyncOperationTaskDefinition: string;
|
|
79
81
|
cluster: string;
|
|
80
82
|
description: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async_operations.d.ts","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"async_operations.d.ts","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";;AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAItC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAQ9D,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAE5F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,KAAK,EACV,wBAAwB,EACxB,2BAA2B,EAC5B,MAAM,SAAS,CAAC;AAYjB,aAAK,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEpF,eAAO,MAAM,sBAAsB,iBACnB,MAAM,KACnB,QAAQ,OAAO,qBAAqB,CAE3B,CAAC;AAEb,eAAO,MAAM,6BAA6B,kBACzB,OAAO,qBAAqB,KAC1C,oBAAoB,EAKtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,YAAY;kCAWO,MAAM;aAC3B,MAAM;sBACG,MAAM;gBACZ,MAAM;QACd,MAAM;mBACK,MAAM;gBACT,MAAM;;qBAED,MAAM;4BAsCxB,CAAC;AAEF,eAAO,MAAM,oBAAoB,WACvB;IACN,YAAY,EAAE,iBAAiB,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qBAAqB,CAAC,EAAE,2BAA2B,CAAA;CACpD,kBACe,wBAAwB,KACvC,QAAQ,QAAQ,iBAAiB,CAAC,CAkCpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,mBAAmB,WACtB;IACN,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4BAA4B,EAAE,MAAM,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,kBAAkB,CAAC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B,CAAC,EAAE,OAAO,CAAC;IACxC,gBAAgB,CAAC,EAAE,MAAM,sBAAsB,CAAA;CAChD,kBACe,wBAAwB,KACvC,QAAQ,QAAQ,iBAAiB,CAAC,CA0FpC,CAAC"}
|
package/dist/async_operations.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
exports.startAsyncOperation = exports.createAsyncOperation = exports.startECSTask = exports.getLambdaEnvironmentVariables = exports.getLambdaConfiguration = void 0;
|
|
4
7
|
const services_1 = require("@cumulus/aws-client/services");
|
|
5
8
|
const db_1 = require("@cumulus/db");
|
|
9
|
+
const logger_1 = __importDefault(require("@cumulus/logger"));
|
|
6
10
|
const uuid_1 = require("uuid");
|
|
7
11
|
const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors');
|
|
8
12
|
const { indexAsyncOperation, } = require('@cumulus/es-client/indexer');
|
|
9
13
|
const { Search, } = require('@cumulus/es-client/search');
|
|
14
|
+
const logger = new logger_1.default({ sender: '@cumulus/async-operation' });
|
|
10
15
|
const getLambdaConfiguration = async (functionName) => (0, services_1.lambda)().getFunctionConfiguration({
|
|
11
16
|
FunctionName: functionName,
|
|
12
17
|
}).promise();
|
|
@@ -129,12 +134,12 @@ exports.createAsyncOperation = createAsyncOperation;
|
|
|
129
134
|
* @memberof AsyncOperation
|
|
130
135
|
*/
|
|
131
136
|
const startAsyncOperation = async (params, AsyncOperation) => {
|
|
132
|
-
var _a;
|
|
137
|
+
var _a, _b;
|
|
133
138
|
const { description, operationType, payload, systemBucket, stackName, dynamoTableName, callerLambdaName, knexConfig = process.env, startEcsTaskFunc = exports.startECSTask, } = params;
|
|
134
139
|
if (!callerLambdaName) {
|
|
135
140
|
throw new MissingRequiredArgument(`callerLambdaName must be specified to start new async operation, received: ${callerLambdaName}`);
|
|
136
141
|
}
|
|
137
|
-
const id = (0, uuid_1.v4)();
|
|
142
|
+
const id = (_a = params.asyncOperationId) !== null && _a !== void 0 ? _a : (0, uuid_1.v4)();
|
|
138
143
|
// Store the payload to S3
|
|
139
144
|
const payloadBucket = systemBucket;
|
|
140
145
|
const payloadKey = `${stackName}/async-operation-payloads/${id}.json`;
|
|
@@ -143,21 +148,49 @@ const startAsyncOperation = async (params, AsyncOperation) => {
|
|
|
143
148
|
Key: payloadKey,
|
|
144
149
|
Body: JSON.stringify(payload),
|
|
145
150
|
});
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
logger.debug(`About to start AsyncOperation: ${id}`);
|
|
152
|
+
let runTaskResponse;
|
|
153
|
+
try {
|
|
154
|
+
runTaskResponse = await startEcsTaskFunc({
|
|
155
|
+
...params,
|
|
156
|
+
id,
|
|
157
|
+
payloadBucket,
|
|
158
|
+
payloadKey,
|
|
159
|
+
});
|
|
160
|
+
if ((runTaskResponse === null || runTaskResponse === void 0 ? void 0 : runTaskResponse.failures) && runTaskResponse.failures.length > 0) {
|
|
161
|
+
throw new EcsStartTaskError(`Failed to start AsyncOperation: ${runTaskResponse.failures[0].reason}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
logger.error(`Failed to start AsyncOperation ${id}`, error);
|
|
166
|
+
const output = {
|
|
167
|
+
name: error.name,
|
|
168
|
+
message: error.message,
|
|
169
|
+
stack: error.stack,
|
|
170
|
+
};
|
|
171
|
+
await (0, exports.createAsyncOperation)({
|
|
172
|
+
createObject: {
|
|
173
|
+
id,
|
|
174
|
+
status: 'RUNNER_FAILED',
|
|
175
|
+
output: JSON.stringify(output),
|
|
176
|
+
description,
|
|
177
|
+
operationType,
|
|
178
|
+
createdAt: Date.now(),
|
|
179
|
+
updatedAt: Date.now(),
|
|
180
|
+
},
|
|
181
|
+
stackName,
|
|
182
|
+
systemBucket,
|
|
183
|
+
dynamoTableName,
|
|
184
|
+
knexConfig,
|
|
185
|
+
}, AsyncOperation);
|
|
186
|
+
throw error;
|
|
155
187
|
}
|
|
188
|
+
logger.debug(`About to create AsyncOperation record: ${id}`);
|
|
156
189
|
return (0, exports.createAsyncOperation)({
|
|
157
190
|
createObject: {
|
|
158
191
|
id,
|
|
159
192
|
status: 'RUNNING',
|
|
160
|
-
taskArn: (
|
|
193
|
+
taskArn: (_b = runTaskResponse === null || runTaskResponse === void 0 ? void 0 : runTaskResponse.tasks) === null || _b === void 0 ? void 0 : _b[0].taskArn,
|
|
161
194
|
description,
|
|
162
195
|
operationType,
|
|
163
196
|
createdAt: Date.now(),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async_operations.js","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"async_operations.js","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";;;;;;AAGA,2DAA+D;AAE/D,oCAKqB;AACrB,6DAAqC;AAErC,+BAAoC;AASpC,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAClF,MAAM,EACJ,mBAAmB,GACpB,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC1C,MAAM,EACJ,MAAM,GACP,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAEzC,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAC;AAI3D,MAAM,sBAAsB,GAAG,KAAK,EACzC,YAAoB,EACmB,EAAE,CAAC,IAAA,iBAAM,GAAE,CAAC,wBAAwB,CAAC;IAC5E,YAAY,EAAE,YAAY;CAC3B,CAAC,CAAC,OAAO,EAAE,CAAC;AAJA,QAAA,sBAAsB,0BAItB;AAEN,MAAM,6BAA6B,GAAG,CAC3C,aAA2C,EACnB,EAAE;;IAAC,OAAA,MAAM,CAAC,OAAO,CAAC,MAAA,MAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,WAAW,0CAAE,SAAS,mCAAI,EAAE,CAAC,CAAC,GAAG,CAC1F,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACR,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;KACd,CAAC,CACH,CAAA;CAAA,CAAC;AAPW,QAAA,6BAA6B,iCAOxC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,4BAA4B,EAC5B,OAAO,EACP,gBAAgB,EAChB,UAAU,EACV,EAAE,EACF,aAAa,EACb,UAAU,EACV,6BAA6B,EAC7B,eAAe,GAWhB,EAA0B,EAAE;;IAC3B,MAAM,OAAO,GAAG;QACd,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,EAAE;QACvC,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,eAAe,EAAE;QACxD,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE;QACzC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,aAAa,IAAI,UAAU,EAAE,EAAE;KAC3C,CAAC;IAC5B,IAAI,QAAQ,GAAG,OAAO,CAAC;IAEvB,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAsB,EAAC,gBAAgB,CAAC,CAAC;IAE1E,IAAI,6BAA6B,EAAE;QACjC,MAAM,YAAY,GAAG,MAAM,IAAA,8BAAsB,EAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,IAAA,qCAA6B,EAAC,YAAY,CAAC,CAAC;QAC/D,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACvC;IAED,OAAO,IAAA,cAAG,GAAE,CAAC,OAAO,CAAC;QACnB,OAAO;QACP,cAAc,EAAE,4BAA4B;QAC5C,UAAU,EAAE,SAAS;QACrB,oBAAoB,EAAE;YACpB,mBAAmB,EAAE;gBACnB,OAAO,EAAE,MAAA,MAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAE,SAAS,0CAAE,SAAS,mCAAI,EAAE;gBACvD,cAAc,EAAE,UAAU;gBAC1B,cAAc,EAAE,MAAA,MAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAE,SAAS,0CAAE,gBAAgB,mCAAI,EAAE;aACtE;SACF;QACD,SAAS,EAAE;YACT,kBAAkB,EAAE;gBAClB;oBACE,IAAI,EAAE,gBAAgB;oBACtB,WAAW,EAAE,QAAQ;iBACtB;aACF;SACF;KACF,CAAC,CAAC,OAAO,EAAE,CAAC;AACf,CAAC,CAAC;AAzDW,QAAA,YAAY,gBAyDvB;AAEK,MAAM,oBAAoB,GAAG,KAAK,EACvC,MAQC,EACD,cAAwC,EACH,EAAE;IACvC,MAAM,EACJ,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,eAAe,EACf,UAAU,GAAG,OAAO,CAAC,GAAG,EACxB,QAAQ,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,EAC5B,qBAAqB,GAAG,IAAI,0BAAqB,EAAE,GACpD,GAAG,MAAM,CAAC;IAEX,MAAM,mBAAmB,GAAG,IAAI,cAAc,CAAC;QAC7C,SAAS;QACT,YAAY;QACZ,SAAS,EAAE,eAAe;KAC3B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,IAAA,kBAAa,EAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IACtD,IAAI,qBAAoD,CAAC;IAEzD,IAAI;QACF,OAAO,MAAM,IAAA,gCAA2B,EAAC,IAAI,EAAE,KAAK,EAAE,GAAqB,EAAE,EAAE;YAC7E,MAAM,cAAc,GAAG,IAAA,uDAAkD,EAAC,YAAY,CAAC,CAAC;YACxF,MAAM,qBAAqB,CAAC,MAAM,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;YACxD,qBAAqB,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACvE,MAAM,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxE,OAAO,qBAAqB,CAAC;QAC/B,CAAC,CAAC,CAAC;KACJ;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,qBAAqB,EAAE;YACzB,MAAM,mBAAmB,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC;SACpE;QACD,MAAM,KAAK,CAAC;KACb;AACH,CAAC,CAAC;AA7CW,QAAA,oBAAoB,wBA6C/B;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACI,MAAM,mBAAmB,GAAG,KAAK,EACtC,MAeC,EACD,cAAwC,EACH,EAAE;;IACvC,MAAM,EACJ,WAAW,EACX,aAAa,EACb,OAAO,EACP,YAAY,EACZ,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,UAAU,GAAG,OAAO,CAAC,GAAG,EACxB,gBAAgB,GAAG,oBAAY,GAChC,GAAG,MAAM,CAAC;IAEX,IAAI,CAAC,gBAAgB,EAAE;QACrB,MAAM,IAAI,uBAAuB,CAAC,8EAA8E,gBAAgB,EAAE,CAAC,CAAC;KACrI;IAED,MAAM,EAAE,GAAG,MAAA,MAAM,CAAC,gBAAgB,mCAAI,IAAA,SAAM,GAAE,CAAC;IAC/C,0BAA0B;IAC1B,MAAM,aAAa,GAAG,YAAY,CAAC;IACnC,MAAM,UAAU,GAAG,GAAG,SAAS,6BAA6B,EAAE,OAAO,CAAC;IAEtE,MAAM,IAAA,aAAE,GAAE,CAAC,SAAS,CAAC;QACnB,MAAM,EAAE,aAAa;QACrB,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC;IAEH,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAC;IACrD,IAAI,eAAe,CAAC;IACpB,IAAI;QACF,eAAe,GAAG,MAAM,gBAAgB,CAAC;YACvC,GAAG,MAAM;YACT,EAAE;YACF,aAAa;YACb,UAAU;SACX,CAAC,CAAC;QAEH,IAAI,CAAA,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,QAAQ,KAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YACpE,MAAM,IAAI,iBAAiB,CACzB,mCAAmC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CACxE,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG;YACb,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC;QACF,MAAM,IAAA,4BAAoB,EACxB;YACE,YAAY,EAAE;gBACZ,EAAE;gBACF,MAAM,EAAE,eAAe;gBACvB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;gBAC9B,WAAW;gBACX,aAAa;gBACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB;YACD,SAAS;YACT,YAAY;YACZ,eAAe;YACf,UAAU;SACX,EACD,cAAc,CACf,CAAC;QACF,MAAM,KAAK,CAAC;KACb;IAED,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE,EAAE,CAAC,CAAC;IAC7D,OAAO,IAAA,4BAAoB,EACzB;QACE,YAAY,EAAE;YACZ,EAAE;YACF,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,MAAA,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,KAAK,0CAAG,CAAC,EAAE,OAAO;YAC5C,WAAW;YACX,aAAa;YACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB;QACD,SAAS;QACT,YAAY;QACZ,eAAe;QACf,UAAU;KACX,EACD,cAAc,CACf,CAAC;AACJ,CAAC,CAAC;AA5GW,QAAA,mBAAmB,uBA4G9B"}
|