@cumulus/async-operations 10.1.1 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/async_operations.d.ts +10 -1
- package/dist/async_operations.d.ts.map +1 -1
- package/dist/async_operations.js +38 -16
- package/dist/async_operations.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types.d.ts +6 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +9 -7
- package/src/async_operations.ts +77 -26
- package/src/types.ts +11 -1
- package/tests/test-async_operations.js +158 -11
|
@@ -4,7 +4,7 @@ import { EnvironmentVariables } from 'aws-sdk/clients/lambda';
|
|
|
4
4
|
import { ApiAsyncOperation, AsyncOperationType } from '@cumulus/types/api/async_operations';
|
|
5
5
|
import type { AWSError } from 'aws-sdk/lib/error';
|
|
6
6
|
import type { PromiseResult } from 'aws-sdk/lib/request';
|
|
7
|
-
import type { AsyncOperationModelClass } from './types';
|
|
7
|
+
import type { AsyncOperationModelClass, AsyncOperationPgModelObject } from './types';
|
|
8
8
|
declare type StartEcsTaskReturnType = Promise<PromiseResult<ECS.RunTaskResponse, AWSError>>;
|
|
9
9
|
export declare const getLambdaConfiguration: (functionName: string) => Promise<Lambda.FunctionConfiguration>;
|
|
10
10
|
export declare const getLambdaEnvironmentVariables: (configuration: Lambda.FunctionConfiguration) => EnvironmentVariables[];
|
|
@@ -39,6 +39,15 @@ export declare const startECSTask: ({ asyncOperationTaskDefinition, cluster, cal
|
|
|
39
39
|
useLambdaEnvironmentVariables?: boolean | undefined;
|
|
40
40
|
dynamoTableName: string;
|
|
41
41
|
}) => StartEcsTaskReturnType;
|
|
42
|
+
export declare const createAsyncOperation: (params: {
|
|
43
|
+
createObject: ApiAsyncOperation;
|
|
44
|
+
stackName: string;
|
|
45
|
+
systemBucket: string;
|
|
46
|
+
dynamoTableName: string;
|
|
47
|
+
knexConfig?: NodeJS.ProcessEnv;
|
|
48
|
+
esClient?: object;
|
|
49
|
+
asyncOperationPgModel?: AsyncOperationPgModelObject;
|
|
50
|
+
}, AsyncOperation: AsyncOperationModelClass) => Promise<Partial<ApiAsyncOperation>>;
|
|
42
51
|
/**
|
|
43
52
|
* Start an AsyncOperation in ECS and store its associate record to DynamoDB
|
|
44
53
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async_operations.d.ts","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"async_operations.d.ts","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAItC,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAO9D,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAE5F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,KAAK,EACV,wBAAwB,EACxB,2BAA2B,EAC5B,MAAM,SAAS,CAAC;AAUjB,aAAK,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEpF,eAAO,MAAM,sBAAsB,iBACnB,MAAM,KACnB,QAAQ,OAAO,qBAAqB,CAE3B,CAAC;AAEb,eAAO,MAAM,6BAA6B,kBACzB,OAAO,qBAAqB,KAC1C,oBAAoB,EAKtB,CAAC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,YAAY;kCAWO,MAAM;aAC3B,MAAM;sBACG,MAAM;gBACZ,MAAM;QACd,MAAM;mBACK,MAAM;gBACT,MAAM;;qBAED,MAAM;4BAsCxB,CAAC;AAEF,eAAO,MAAM,oBAAoB,WACvB;IACN,YAAY,EAAE,iBAAiB,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qBAAqB,CAAC,EAAE,2BAA2B,CAAA;CACpD,kBACe,wBAAwB,KACvC,QAAQ,QAAQ,iBAAiB,CAAC,CAkCpC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,eAAO,MAAM,mBAAmB,WACtB;IACN,4BAA4B,EAAE,MAAM,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,UAAU,CAAC;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,kBAAkB,CAAC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,6BAA6B,CAAC,EAAE,OAAO,CAAC;IACxC,gBAAgB,CAAC,EAAE,MAAM,sBAAsB,CAAA;CAChD,kBACe,wBAAwB,KACvC,QAAQ,QAAQ,iBAAiB,CAAC,CA4DpC,CAAC"}
|
package/dist/async_operations.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.startAsyncOperation = exports.startECSTask = exports.getLambdaEnvironmentVariables = exports.getLambdaConfiguration = void 0;
|
|
3
|
+
exports.startAsyncOperation = exports.createAsyncOperation = exports.startECSTask = exports.getLambdaEnvironmentVariables = exports.getLambdaConfiguration = void 0;
|
|
4
4
|
const services_1 = require("@cumulus/aws-client/services");
|
|
5
5
|
const db_1 = require("@cumulus/db");
|
|
6
6
|
const uuid_1 = require("uuid");
|
|
7
7
|
const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors');
|
|
8
|
+
const { indexAsyncOperation, } = require('@cumulus/es-client/indexer');
|
|
9
|
+
const { Search, } = require('@cumulus/es-client/search');
|
|
8
10
|
const getLambdaConfiguration = async (functionName) => (0, services_1.lambda)().getFunctionConfiguration({
|
|
9
11
|
FunctionName: functionName,
|
|
10
12
|
}).promise();
|
|
@@ -74,6 +76,32 @@ const startECSTask = async ({ asyncOperationTaskDefinition, cluster, callerLambd
|
|
|
74
76
|
}).promise();
|
|
75
77
|
};
|
|
76
78
|
exports.startECSTask = startECSTask;
|
|
79
|
+
const createAsyncOperation = async (params, AsyncOperation) => {
|
|
80
|
+
const { createObject, stackName, systemBucket, dynamoTableName, knexConfig = process.env, esClient = await Search.es(), asyncOperationPgModel = new db_1.AsyncOperationPgModel(), } = params;
|
|
81
|
+
const asyncOperationModel = new AsyncOperation({
|
|
82
|
+
stackName,
|
|
83
|
+
systemBucket,
|
|
84
|
+
tableName: dynamoTableName,
|
|
85
|
+
});
|
|
86
|
+
const knex = await (0, db_1.getKnexClient)({ env: knexConfig });
|
|
87
|
+
let createdAsyncOperation;
|
|
88
|
+
try {
|
|
89
|
+
return await (0, db_1.createRejectableTransaction)(knex, async (trx) => {
|
|
90
|
+
const pgCreateObject = (0, db_1.translateApiAsyncOperationToPostgresAsyncOperation)(createObject);
|
|
91
|
+
await asyncOperationPgModel.create(trx, pgCreateObject);
|
|
92
|
+
createdAsyncOperation = await asyncOperationModel.create(createObject);
|
|
93
|
+
await indexAsyncOperation(esClient, createObject, process.env.ES_INDEX);
|
|
94
|
+
return createdAsyncOperation;
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (createdAsyncOperation) {
|
|
99
|
+
await asyncOperationModel.delete({ id: createdAsyncOperation.id });
|
|
100
|
+
}
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
exports.createAsyncOperation = createAsyncOperation;
|
|
77
105
|
/**
|
|
78
106
|
* Start an AsyncOperation in ECS and store its associate record to DynamoDB
|
|
79
107
|
*
|
|
@@ -101,6 +129,7 @@ exports.startECSTask = startECSTask;
|
|
|
101
129
|
* @memberof AsyncOperation
|
|
102
130
|
*/
|
|
103
131
|
const startAsyncOperation = async (params, AsyncOperation) => {
|
|
132
|
+
var _a;
|
|
104
133
|
const { description, operationType, payload, systemBucket, stackName, dynamoTableName, callerLambdaName, knexConfig = process.env, startEcsTaskFunc = exports.startECSTask, } = params;
|
|
105
134
|
if (!callerLambdaName) {
|
|
106
135
|
throw new MissingRequiredArgument(`callerLambdaName must be specified to start new async operation, received: ${callerLambdaName}`);
|
|
@@ -124,16 +153,8 @@ const startAsyncOperation = async (params, AsyncOperation) => {
|
|
|
124
153
|
if ((runTaskResponse === null || runTaskResponse === void 0 ? void 0 : runTaskResponse.failures) && runTaskResponse.failures.length > 0) {
|
|
125
154
|
throw new EcsStartTaskError(`Failed to start AsyncOperation: ${runTaskResponse.failures[0].reason}`);
|
|
126
155
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
systemBucket,
|
|
130
|
-
tableName: dynamoTableName,
|
|
131
|
-
});
|
|
132
|
-
const asyncOperationPgModel = new db_1.AsyncOperationPgModel();
|
|
133
|
-
const knex = await (0, db_1.getKnexClient)({ env: knexConfig });
|
|
134
|
-
return (0, db_1.createRejectableTransaction)(knex, async (trx) => {
|
|
135
|
-
var _a;
|
|
136
|
-
const createObject = {
|
|
156
|
+
return (0, exports.createAsyncOperation)({
|
|
157
|
+
createObject: {
|
|
137
158
|
id,
|
|
138
159
|
status: 'RUNNING',
|
|
139
160
|
taskArn: (_a = runTaskResponse === null || runTaskResponse === void 0 ? void 0 : runTaskResponse.tasks) === null || _a === void 0 ? void 0 : _a[0].taskArn,
|
|
@@ -141,11 +162,12 @@ const startAsyncOperation = async (params, AsyncOperation) => {
|
|
|
141
162
|
operationType,
|
|
142
163
|
createdAt: Date.now(),
|
|
143
164
|
updatedAt: Date.now(),
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
165
|
+
},
|
|
166
|
+
stackName,
|
|
167
|
+
systemBucket,
|
|
168
|
+
dynamoTableName,
|
|
169
|
+
knexConfig,
|
|
170
|
+
}, AsyncOperation);
|
|
149
171
|
};
|
|
150
172
|
exports.startAsyncOperation = startAsyncOperation;
|
|
151
173
|
//# sourceMappingURL=async_operations.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"async_operations.js","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"async_operations.js","sourceRoot":"","sources":["../src/async_operations.ts"],"names":[],"mappings":";;;AAGA,2DAA+D;AAE/D,oCAKqB;AAErB,+BAAoC;AASpC,MAAM,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAClF,MAAM,EACJ,mBAAmB,GACpB,GAAG,OAAO,CAAC,4BAA4B,CAAC,CAAC;AAC1C,MAAM,EACJ,MAAM,GACP,GAAG,OAAO,CAAC,2BAA2B,CAAC,CAAC;AAIlC,MAAM,sBAAsB,GAAG,KAAK,EACzC,YAAoB,EACmB,EAAE,CAAC,IAAA,iBAAM,GAAE,CAAC,wBAAwB,CAAC;IAC5E,YAAY,EAAE,YAAY;CAC3B,CAAC,CAAC,OAAO,EAAE,CAAC;AAJA,QAAA,sBAAsB,0BAItB;AAEN,MAAM,6BAA6B,GAAG,CAC3C,aAA2C,EACnB,EAAE;;IAAC,OAAA,MAAM,CAAC,OAAO,CAAC,MAAA,MAAA,aAAa,aAAb,aAAa,uBAAb,aAAa,CAAE,WAAW,0CAAE,SAAS,mCAAI,EAAE,CAAC,CAAC,GAAG,CAC1F,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACR,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACZ,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;KACd,CAAC,CACH,CAAA;CAAA,CAAC;AAPW,QAAA,6BAA6B,iCAOxC;AAEF;;;;;;;;;;;;;;;;;;;GAmBG;AACI,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,4BAA4B,EAC5B,OAAO,EACP,gBAAgB,EAChB,UAAU,EACV,EAAE,EACF,aAAa,EACb,UAAU,EACV,6BAA6B,EAC7B,eAAe,GAWhB,EAA0B,EAAE;;IAC3B,MAAM,OAAO,GAAG;QACd,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,EAAE;QACvC,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,eAAe,EAAE;QACxD,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE;QACzC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,aAAa,IAAI,UAAU,EAAE,EAAE;KAC3C,CAAC;IAC5B,IAAI,QAAQ,GAAG,OAAO,CAAC;IAEvB,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAsB,EAAC,gBAAgB,CAAC,CAAC;IAE1E,IAAI,6BAA6B,EAAE;QACjC,MAAM,YAAY,GAAG,MAAM,IAAA,8BAAsB,EAAC,UAAU,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,IAAA,qCAA6B,EAAC,YAAY,CAAC,CAAC;QAC/D,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;KACvC;IAED,OAAO,IAAA,cAAG,GAAE,CAAC,OAAO,CAAC;QACnB,OAAO;QACP,cAAc,EAAE,4BAA4B;QAC5C,UAAU,EAAE,SAAS;QACrB,oBAAoB,EAAE;YACpB,mBAAmB,EAAE;gBACnB,OAAO,EAAE,MAAA,MAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAE,SAAS,0CAAE,SAAS,mCAAI,EAAE;gBACvD,cAAc,EAAE,UAAU;gBAC1B,cAAc,EAAE,MAAA,MAAA,kBAAkB,aAAlB,kBAAkB,uBAAlB,kBAAkB,CAAE,SAAS,0CAAE,gBAAgB,mCAAI,EAAE;aACtE;SACF;QACD,SAAS,EAAE;YACT,kBAAkB,EAAE;gBAClB;oBACE,IAAI,EAAE,gBAAgB;oBACtB,WAAW,EAAE,QAAQ;iBACtB;aACF;SACF;KACF,CAAC,CAAC,OAAO,EAAE,CAAC;AACf,CAAC,CAAC;AAzDW,QAAA,YAAY,gBAyDvB;AAEK,MAAM,oBAAoB,GAAG,KAAK,EACvC,MAQC,EACD,cAAwC,EACH,EAAE;IACvC,MAAM,EACJ,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,eAAe,EACf,UAAU,GAAG,OAAO,CAAC,GAAG,EACxB,QAAQ,GAAG,MAAM,MAAM,CAAC,EAAE,EAAE,EAC5B,qBAAqB,GAAG,IAAI,0BAAqB,EAAE,GACpD,GAAG,MAAM,CAAC;IAEX,MAAM,mBAAmB,GAAG,IAAI,cAAc,CAAC;QAC7C,SAAS;QACT,YAAY;QACZ,SAAS,EAAE,eAAe;KAC3B,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,IAAA,kBAAa,EAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IACtD,IAAI,qBAAoD,CAAC;IAEzD,IAAI;QACF,OAAO,MAAM,IAAA,gCAA2B,EAAC,IAAI,EAAE,KAAK,EAAE,GAAqB,EAAE,EAAE;YAC7E,MAAM,cAAc,GAAG,IAAA,uDAAkD,EAAC,YAAY,CAAC,CAAC;YACxF,MAAM,qBAAqB,CAAC,MAAM,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;YACxD,qBAAqB,GAAG,MAAM,mBAAmB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACvE,MAAM,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACxE,OAAO,qBAAqB,CAAC;QAC/B,CAAC,CAAC,CAAC;KACJ;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,qBAAqB,EAAE;YACzB,MAAM,mBAAmB,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,qBAAqB,CAAC,EAAE,EAAE,CAAC,CAAC;SACpE;QACD,MAAM,KAAK,CAAC;KACb;AACH,CAAC,CAAC;AA7CW,QAAA,oBAAoB,wBA6C/B;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACI,MAAM,mBAAmB,GAAG,KAAK,EACtC,MAcC,EACD,cAAwC,EACH,EAAE;;IACvC,MAAM,EACJ,WAAW,EACX,aAAa,EACb,OAAO,EACP,YAAY,EACZ,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,UAAU,GAAG,OAAO,CAAC,GAAG,EACxB,gBAAgB,GAAG,oBAAY,GAChC,GAAG,MAAM,CAAC;IAEX,IAAI,CAAC,gBAAgB,EAAE;QACrB,MAAM,IAAI,uBAAuB,CAAC,8EAA8E,gBAAgB,EAAE,CAAC,CAAC;KACrI;IAED,MAAM,EAAE,GAAG,IAAA,SAAM,GAAE,CAAC;IACpB,0BAA0B;IAC1B,MAAM,aAAa,GAAG,YAAY,CAAC;IACnC,MAAM,UAAU,GAAG,GAAG,SAAS,6BAA6B,EAAE,OAAO,CAAC;IAEtE,MAAM,IAAA,aAAE,GAAE,CAAC,SAAS,CAAC;QACnB,MAAM,EAAE,aAAa;QACrB,GAAG,EAAE,UAAU;QACf,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC,OAAO,EAAE,CAAC;IAEb,wBAAwB;IACxB,MAAM,eAAe,GAAG,MAAM,gBAAgB,CAAC;QAC7C,GAAG,MAAM;QACT,EAAE;QACF,aAAa;QACb,UAAU;KACX,CAAC,CAAC;IAEH,IAAI,CAAA,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,QAAQ,KAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;QACpE,MAAM,IAAI,iBAAiB,CACzB,mCAAmC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CACxE,CAAC;KACH;IAED,OAAO,IAAA,4BAAoB,EACzB;QACE,YAAY,EAAE;YACZ,EAAE;YACF,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,MAAA,eAAe,aAAf,eAAe,uBAAf,eAAe,CAAE,KAAK,0CAAG,CAAC,EAAE,OAAO;YAC5C,WAAW;YACX,aAAa;YACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB;QACD,SAAS;QACT,YAAY;QACZ,eAAe;QACf,UAAU;KACX,EACD,cAAc,CACf,CAAC;AACJ,CAAC,CAAC;AA7EW,QAAA,mBAAmB,uBA6E9B"}
|