@cumulus/async-operations 9.8.0-alpha.0 → 10.0.0

package/dist/types.d.ts

@@ -0,0 +1,9 @@
+export interface AsyncOperationModelClass {
+    new (params: {
+        stackName: string;
+        systemBucket: string;
+        tableName?: string;
+    }): any;
+    create(...args: any): any | any[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cumulus/async-operations",
-  "version": "9.8.0-alpha.0",
+  "version": "10.0.0",
   "description": "Cumulus Core internal async operations module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -28,15 +28,16 @@
   "author": "Cumulus Authors",
   "license": "Apache-2.0",
   "dependencies": {
-    "@cumulus/aws-client": "9.7.0",
-    "@cumulus/db": "9.8.0-alpha.0",
-    "@cumulus/errors": "9.7.0",
-    "@cumulus/types": "9.7.0",
+    "@cumulus/aws-client": "10.0.0",
+    "@cumulus/db": "10.0.0",
+    "@cumulus/errors": "10.0.0",
+    "@cumulus/types": "10.0.0",
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@cumulus/common": "9.7.0",
+    "@cumulus/common": "10.0.0",
     "@types/aws-sdk": "2.7.0",
     "@types/uuid": "^8.0.0"
-  }
+  },
+  "gitHead": "7c6d2d1cd79b57d6943bbc3d898d0cf975b543b1"
 }

package/src/async_operations.ts

@@ -1,4 +1,4 @@
-import { ECS } from 'aws-sdk';
+import { ECS, Lambda } from 'aws-sdk';
 import { ecs, s3, lambda } from '@cumulus/aws-client/services';
 import { EnvironmentVariables } from 'aws-sdk/clients/lambda';
 import {
@@ -6,6 +6,7 @@ import {
   translateApiAsyncOperationToPostgresAsyncOperation,
   AsyncOperationPgModel,
   createRejectableTransaction,
+  Knex,
 } from '@cumulus/db';
 import { ApiAsyncOperation, AsyncOperationType } from '@cumulus/types/api/async_operations';
 import { v4 as uuidv4 } from 'uuid';
@@ -14,22 +15,24 @@ import type { PromiseResult } from 'aws-sdk/lib/request';
 
 import type { AsyncOperationModelClass } from './types';
 
-const { EcsStartTaskError } = require('@cumulus/errors');
+const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors');
 
 type StartEcsTaskReturnType = Promise<PromiseResult<ECS.RunTaskResponse, AWSError>>;
 
-export const getLambdaEnvironmentVariables = async (
+export const getLambdaConfiguration = async (
   functionName: string
-): Promise<EnvironmentVariables[]> => {
-  const lambdaConfig = await lambda().getFunctionConfiguration({
-    FunctionName: functionName,
-  }).promise();
-
-  return Object.entries(lambdaConfig?.Environment?.Variables ?? {}).map((obj) => ({
+): Promise<Lambda.FunctionConfiguration> => lambda().getFunctionConfiguration({
+  FunctionName: functionName,
+}).promise();
+
+export const getLambdaEnvironmentVariables = (
+  configuration: Lambda.FunctionConfiguration
+): EnvironmentVariables[] => Object.entries(configuration?.Environment?.Variables ?? {}).map(
+  (obj) => ({
     name: obj[0],
     value: obj[1],
-  }));
-};
+  })
+);
 
 /**
  * Start an ECS task for the async operation.
@@ -37,6 +40,8 @@ export const getLambdaEnvironmentVariables = async (
  * @param {Object} params
  * @param {string} params.asyncOperationTaskDefinition - ARN for the task definition
  * @param {string} params.cluster - ARN for the ECS cluster to use for the task
+ * @param {string} params.callerlambdaName
+ *   Environment variable for Lambda name that is initiating the ECS task
  * @param {string} params.lambdaName
  *   Environment variable for Lambda name that will be run by the ECS task
  * @param {string} params.id - the Async operation ID
@@ -52,6 +57,7 @@ export const getLambdaEnvironmentVariables = async (
 export const startECSTask = async ({
   asyncOperationTaskDefinition,
   cluster,
+  callerLambdaName,
   lambdaName,
   id,
   payloadBucket,
@@ -61,6 +67,7 @@ export const startECSTask = async ({
 }: {
   asyncOperationTaskDefinition: string,
   cluster: string,
+  callerLambdaName: string,
   lambdaName: string,
   id: string,
   payloadBucket: string,
@@ -76,15 +83,25 @@ export const startECSTask = async ({
   ] as EnvironmentVariables[];
   let taskVars = envVars;
 
+  const callerLambdaConfig = await getLambdaConfiguration(callerLambdaName);
+
   if (useLambdaEnvironmentVariables) {
-    const lambdaVars = await getLambdaEnvironmentVariables(lambdaName);
+    const lambdaConfig = await getLambdaConfiguration(lambdaName);
+    const lambdaVars = getLambdaEnvironmentVariables(lambdaConfig);
     taskVars = envVars.concat(lambdaVars);
   }
 
   return ecs().runTask({
     cluster,
     taskDefinition: asyncOperationTaskDefinition,
-    launchType: 'EC2',
+    launchType: 'FARGATE',
+    networkConfiguration: {
+      awsvpcConfiguration: {
+        subnets: callerLambdaConfig?.VpcConfig?.SubnetIds ?? [],
+        assignPublicIp: 'DISABLED',
+        securityGroups: callerLambdaConfig?.VpcConfig?.SecurityGroupIds ?? [],
+      },
+    },
     overrides: {
       containerOverrides: [
         {
@@ -107,6 +124,7 @@ export const startECSTask = async ({
  * @param {string} params.dynamoTableName - the dynamo async operations table to
  * write records to
  * @param {Object} params.knexConfig - Object with Knex configuration keys
+ * @param {string} params.callerLambdaName - the name of the Lambda initiating the ECS task
  * @param {string} params.lambdaName - the name of the Lambda task to be run
  * @param {string} params.operationType - the type of async operation to run
  * @param {Object|Array} params.payload - the event to be passed to the lambda task.
@@ -121,20 +139,23 @@ export const startECSTask = async ({
  * @returns {Promise<Object>} - an AsyncOperation record
  * @memberof AsyncOperation
  */
-export const startAsyncOperation = async (params: {
-  asyncOperationTaskDefinition: string,
-  cluster: string,
-  description: string,
-  dynamoTableName: string,
-  knexConfig?: NodeJS.ProcessEnv,
-  lambdaName: string,
-  operationType: AsyncOperationType,
-  payload: unknown,
-  stackName: string,
-  systemBucket: string,
-  useLambdaEnvironmentVariables?: boolean,
-  startEcsTaskFunc?: () => StartEcsTaskReturnType
-}, AsyncOperation: AsyncOperationModelClass
+export const startAsyncOperation = async (
+  params: {
+    asyncOperationTaskDefinition: string,
+    cluster: string,
+    description: string,
+    dynamoTableName: string,
+    knexConfig?: NodeJS.ProcessEnv,
+    callerLambdaName: string,
+    lambdaName: string,
+    operationType: AsyncOperationType,
+    payload: unknown,
+    stackName: string,
+    systemBucket: string,
+    useLambdaEnvironmentVariables?: boolean,
+    startEcsTaskFunc?: () => StartEcsTaskReturnType
+  },
+  AsyncOperation: AsyncOperationModelClass
 ): Promise<Partial<ApiAsyncOperation>> => {
   const {
     description,
@@ -143,10 +164,15 @@ export const startAsyncOperation = async (params: {
     systemBucket,
     stackName,
     dynamoTableName,
+    callerLambdaName,
     knexConfig = process.env,
     startEcsTaskFunc = startECSTask,
   } = params;
 
+  if (!callerLambdaName) {
+    throw new MissingRequiredArgument(`callerLambdaName must be specified to start new async operation, received: ${callerLambdaName}`);
+  }
+
   const id = uuidv4();
   // Store the payload to S3
   const payloadBucket = systemBucket;
@@ -180,7 +206,7 @@ export const startAsyncOperation = async (params: {
   const asyncOperationPgModel = new AsyncOperationPgModel();
 
   const knex = await getKnexClient({ env: knexConfig });
-  return createRejectableTransaction<ApiAsyncOperation>(knex, async (trx: any) => {
+  return createRejectableTransaction<ApiAsyncOperation>(knex, async (trx: Knex.Transaction) => {
     const createObject: ApiAsyncOperation = {
       id,
       status: 'RUNNING',

package/src/index.ts

@@ -1,2 +1,2 @@
-import { startAsyncOperation } from './async_operations';
-module.exports = { startAsyncOperation };
+import * as asyncOperations from './async_operations';
+module.exports = asyncOperations;

package/tests/test-async_operations.js

@@ -18,9 +18,13 @@ const {
   AsyncOperationPgModel,
   migrationDir,
 } = require('@cumulus/db');
-const { EcsStartTaskError } = require('@cumulus/errors');
+const { EcsStartTaskError, MissingRequiredArgument } = require('@cumulus/errors');
 
-const { getLambdaEnvironmentVariables, startAsyncOperation } = require('../dist/async_operations');
+const {
+  getLambdaConfiguration,
+  getLambdaEnvironmentVariables,
+  startAsyncOperation,
+} = require('../dist/async_operations');
 
 const dynamoTableName = 'notUsedDynamoTableName';
 
@@ -56,15 +60,17 @@ test.before(async (t) => {
     };
   };
 
-  sinon.stub(lambda(), 'getFunctionConfiguration').returns({
-    promise: () => Promise.resolve({
-      Environment: {
-        Variables: {
-          ES_HOST: 'es-host',
-          AsyncOperationsTable: 'async-operations-table',
-        },
+  t.context.functionConfig = {
+    Environment: {
+      Variables: {
+        ES_HOST: 'es-host',
+        AsyncOperationsTable: 'async-operations-table',
       },
-    }),
+    },
+  };
+
+  sinon.stub(lambda(), 'getFunctionConfiguration').returns({
+    promise: () => Promise.resolve(t.context.functionConfig),
   });
 
   t.context.asyncOperationPgModel = new AsyncOperationPgModel();
@@ -96,6 +102,7 @@ test.serial('startAsyncOperation uploads the payload to S3', async (t) => {
   const { id } = await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description: randomString(),
     operationType: 'ES Index',
@@ -128,6 +135,7 @@ test.serial('The AsyncOperation start method starts an ECS task with the correct
 
   const asyncOperationTaskDefinition = randomString();
   const cluster = randomString();
+  const callerLambdaName = randomString();
   const lambdaName = randomString();
   const payload = { x: randomString() };
   const stackName = randomString();
@@ -136,6 +144,7 @@ test.serial('The AsyncOperation start method starts an ECS task with the correct
     asyncOperationTaskDefinition,
     cluster,
     lambdaName,
+    callerLambdaName,
     description: randomString(),
     operationType: 'ES Index',
     payload,
@@ -148,7 +157,7 @@ test.serial('The AsyncOperation start method starts an ECS task with the correct
 
   t.is(stubbedEcsRunTaskParams.cluster, cluster);
   t.is(stubbedEcsRunTaskParams.taskDefinition, asyncOperationTaskDefinition);
-  t.is(stubbedEcsRunTaskParams.launchType, 'EC2');
+  t.is(stubbedEcsRunTaskParams.launchType, 'FARGATE');
 
   const environmentOverrides = {};
   stubbedEcsRunTaskParams.overrides.containerOverrides[0].environment.forEach((env) => {
@@ -176,6 +185,7 @@ test.serial('The startAsyncOperation method throws error if it is unable to crea
   await t.throwsAsync(startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description: randomString(),
     operationType: 'ES Index',
@@ -206,6 +216,7 @@ test('startAsyncOperation calls Dynamo model create method', async (t) => {
   const result = await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description,
     operationType: 'ES Index',
@@ -248,6 +259,7 @@ test.serial('The startAsyncOperation writes records to the databases', async (t)
   const { id } = await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description,
     operationType,
@@ -296,6 +308,7 @@ test.serial('The startAsyncOperation writes records to the databases with correc
   const { id } = await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description,
     operationType,
@@ -332,6 +345,7 @@ test.serial('The startAsyncOperation method returns the newly-generated record',
   const results = await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description: randomString(),
     operationType: 'ES Index',
@@ -345,8 +359,42 @@ test.serial('The startAsyncOperation method returns the newly-generated record',
   t.is(results.taskArn, taskArn);
 });
 
-test('getLambdaEnvironmentVariables returns expected environment variables', async (t) => {
-  const vars = await getLambdaEnvironmentVariables('name');
+test.serial('The startAsyncOperation method throws error if callerLambdaName parameter is missing', async (t) => {
+  const stubbedAsyncOperationsModel = class {
+    create = sinon.stub();
+  };
+
+  stubbedEcsRunTaskParams = {};
+  stubbedEcsRunTaskResult = {
+    tasks: [{ taskArn: randomString() }],
+    failures: [],
+  };
+
+  await t.throwsAsync(
+    startAsyncOperation({
+      asyncOperationTaskDefinition: randomString(),
+      cluster: randomString,
+      lambdaName: randomString,
+      description: randomString(),
+      operationType: 'ES Index',
+      payload: { x: randomString() },
+      stackName: randomString,
+      dynamoTableName: dynamoTableName,
+      knexConfig: knexConfig,
+      systemBucket,
+      useLambdaEnvironmentVariables: true,
+    }, stubbedAsyncOperationsModel),
+    { instanceOf: MissingRequiredArgument }
+  );
+});
+
+test('getLambdaConfiguration returns expected configuration', async (t) => {
+  const config = await getLambdaConfiguration('name');
+  t.deepEqual(config, t.context.functionConfig);
+});
+
+test('getLambdaEnvironmentVariables returns expected environment variables', (t) => {
+  const vars = getLambdaEnvironmentVariables(t.context.functionConfig);
 
   t.deepEqual(new Set(vars), new Set([
     { name: 'ES_HOST', value: 'es-host' },
@@ -370,6 +418,7 @@ test.serial('ECS task params contain lambda environment variables when useLambda
   await startAsyncOperation({
     asyncOperationTaskDefinition: randomString(),
     cluster: randomString(),
+    callerLambdaName: randomString(),
     lambdaName: randomString(),
     description: randomString(),
     operationType: 'ES Index',