@cumulus/async-operations 12.0.0 → 13.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/async-operations",
-  "version": "12.0.0",
+  "version": "13.0.0",
   "description": "Cumulus Core internal async operations module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -28,18 +28,18 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "12.0.0",
-    "@cumulus/db": "12.0.0",
-    "@cumulus/errors": "12.0.0",
-    "@cumulus/es-client": "12.0.0",
-    "@cumulus/types": "12.0.0",
+    "@cumulus/aws-client": "13.0.0",
+    "@cumulus/db": "13.0.0",
+    "@cumulus/errors": "13.0.0",
+    "@cumulus/es-client": "13.0.0",
+    "@cumulus/types": "13.0.0",
     "knex": "0.95.15",
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@cumulus/common": "12.0.0",
+    "@cumulus/common": "13.0.0",
     "@types/aws-sdk": "2.7.0",
     "@types/uuid": "^8.0.0"
   },
-  "gitHead": "2d84198510f37d64444715736b8b08360f61c766"
+  "gitHead": "ba43ea3bfdfcacfff052770ed2fd357eb9dadb58"
 }

package/src/async_operations.ts

@@ -9,6 +9,7 @@ import {
   AsyncOperationPgModel,
   createRejectableTransaction,
 } from '@cumulus/db';
+import Logger from '@cumulus/logger';
 import { ApiAsyncOperation, AsyncOperationType } from '@cumulus/types/api/async_operations';
 import { v4 as uuidv4 } from 'uuid';
 import type { AWSError } from 'aws-sdk/lib/error';
@@ -27,6 +28,8 @@ const {
   Search,
 } = require('@cumulus/es-client/search');
 
+const logger = new Logger({ sender: '@cumulus/async-operation' });
+
 type StartEcsTaskReturnType = Promise<PromiseResult<ECS.RunTaskResponse, AWSError>>;
 
 export const getLambdaConfiguration = async (
@@ -198,6 +201,7 @@ export const createAsyncOperation = async (
  */
 export const startAsyncOperation = async (
   params: {
+    asyncOperationId?: string,
     asyncOperationTaskDefinition: string,
     cluster: string,
     description: string,
@@ -230,7 +234,7 @@ export const startAsyncOperation = async (
     throw new MissingRequiredArgument(`callerLambdaName must be specified to start new async operation, received: ${callerLambdaName}`);
   }
 
-  const id = uuidv4();
+  const id = params.asyncOperationId ?? uuidv4();
   // Store the payload to S3
   const payloadBucket = systemBucket;
   const payloadKey = `${stackName}/async-operation-payloads/${id}.json`;
@@ -241,20 +245,50 @@ export const startAsyncOperation = async (
     Body: JSON.stringify(payload),
   });
 
-  // Start the task in ECS
-  const runTaskResponse = await startEcsTaskFunc({
-    ...params,
-    id,
-    payloadBucket,
-    payloadKey,
-  });
+  logger.debug(`About to start AsyncOperation: ${id}`);
+  let runTaskResponse;
+  try {
+    runTaskResponse = await startEcsTaskFunc({
+      ...params,
+      id,
+      payloadBucket,
+      payloadKey,
+    });
 
-  if (runTaskResponse?.failures && runTaskResponse.failures.length > 0) {
-    throw new EcsStartTaskError(
-      `Failed to start AsyncOperation: ${runTaskResponse.failures[0].reason}`
+    if (runTaskResponse?.failures && runTaskResponse.failures.length > 0) {
+      throw new EcsStartTaskError(
+        `Failed to start AsyncOperation: ${runTaskResponse.failures[0].reason}`
+      );
+    }
+  } catch (error) {
+    logger.error(`Failed to start AsyncOperation ${id}`, error);
+    const output = {
+      name: error.name,
+      message: error.message,
+      stack: error.stack,
+    };
+    await createAsyncOperation(
+      {
+        createObject: {
+          id,
+          status: 'RUNNER_FAILED',
+          output: JSON.stringify(output),
+          description,
+          operationType,
+          createdAt: Date.now(),
+          updatedAt: Date.now(),
+        },
+        stackName,
+        systemBucket,
+        dynamoTableName,
+        knexConfig,
+      },
+      AsyncOperation
     );
+    throw error;
   }
 
+  logger.debug(`About to create AsyncOperation record: ${id}`);
   return createAsyncOperation(
     {
       createObject: {

package/tests/test-async_operations.js

@@ -207,8 +207,61 @@ test.serial('The AsyncOperation start method starts an ECS task with the correct
   t.is(environmentOverrides.payloadUrl, `s3://${systemBucket}/${stackName}/async-operation-payloads/${id}.json`);
 });
 
-test.serial('The startAsyncOperation method throws error if it is unable to create an ECS task', async (t) => {
-  const createSpy = sinon.spy((obj) => ({ id: obj.id }));
+test.serial('The AsyncOperation start method starts an ECS task with the asyncOperationId passed in', async (t) => {
+  const createSpy = sinon.spy((obj) => obj);
+  const stubbedAsyncOperationsModel = class {
+    create = createSpy;
+  };
+
+  stubbedEcsRunTaskParams = {};
+  stubbedEcsRunTaskResult = {
+    tasks: [{ taskArn: randomString() }],
+    failures: [],
+  };
+
+  const asyncOperationId = uuidv4();
+  const asyncOperationTaskDefinition = randomString();
+  const cluster = randomString();
+  const callerLambdaName = randomString();
+  const lambdaName = randomString();
+  const payload = { x: randomString() };
+  const stackName = randomString();
+
+  const { id } = await startAsyncOperation({
+    asyncOperationId,
+    asyncOperationTaskDefinition,
+    cluster,
+    lambdaName,
+    callerLambdaName,
+    description: randomString(),
+    operationType: 'ES Index',
+    payload,
+    stackName,
+    dynamoTableName: dynamoTableName,
+    knexConfig: knexConfig,
+    systemBucket,
+    useLambdaEnvironmentVariables: true,
+  }, stubbedAsyncOperationsModel);
+
+  t.is(stubbedEcsRunTaskParams.cluster, cluster);
+  t.is(stubbedEcsRunTaskParams.taskDefinition, asyncOperationTaskDefinition);
+  t.is(stubbedEcsRunTaskParams.launchType, 'FARGATE');
+
+  const environmentOverrides = {};
+  stubbedEcsRunTaskParams.overrides.containerOverrides[0].environment.forEach((env) => {
+    environmentOverrides[env.name] = env.value;
+  });
+
+  t.is(id, asyncOperationId);
+  t.is(environmentOverrides.asyncOperationId, asyncOperationId);
+  t.is(environmentOverrides.asyncOperationsTable, dynamoTableName);
+  t.is(environmentOverrides.lambdaName, lambdaName);
+  t.is(environmentOverrides.payloadUrl, `s3://${systemBucket}/${stackName}/async-operation-payloads/${asyncOperationId}.json`);
+});
+
+test.serial('The startAsyncOperation method throws error and calls database model create method '
+  + 'when it is unable to create an ECS task', async (t) => {
+  const createSpy = sinon.spy((obj) => obj);
   const stubbedAsyncOperationsModel = class {
     create = createSpy;
   };
@@ -219,7 +272,8 @@ test.serial('The startAsyncOperation method throws error if it is unable to crea
   };
   const stackName = randomString();
 
-  await t.throwsAsync(startAsyncOperation({
+  const asyncOperationParams = {
+    asyncOperationId: uuidv4(),
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
     callerLambdaName: randomString(),
@@ -231,10 +285,30 @@ test.serial('The startAsyncOperation method throws error if it is unable to crea
     dynamoTableName: dynamoTableName,
     knexConfig: knexConfig,
     systemBucket,
-  }, stubbedAsyncOperationsModel), {
+  };
+  const expectedErrorThrown = {
     instanceOf: EcsStartTaskError,
     message: 'Failed to start AsyncOperation: out of cheese',
-  });
+  };
+  await t.throwsAsync(
+    startAsyncOperation(asyncOperationParams, stubbedAsyncOperationsModel),
+    expectedErrorThrown
+  );
+
+  const spyCall = createSpy.getCall(0).args[0];
+
+  const expected = {
+    id: asyncOperationParams.asyncOperationId,
+    description: asyncOperationParams.description,
+    operationType: asyncOperationParams.operationType,
+    status: 'RUNNER_FAILED',
+  };
+
+  t.like(spyCall, expected);
+  t.deepEqual(omit(spyCall, ['createdAt', 'updatedAt', 'output']), expected);
+  t.is(spyCall.id, asyncOperationParams.asyncOperationId);
+  const output = JSON.parse(spyCall.output || {});
+  t.like(output, { name: 'EcsStartTaskError', message: expectedErrorThrown.message });
 });
 
 test('startAsyncOperation calls Dynamo model create method', async (t) => {