@cumulus/message 10.0.1 → 10.1.1

package/src/Collections.ts

@@ -1,6 +1,7 @@
 'use strict';
 
 import { Message } from '@cumulus/types';
+const collectionIdSeparator = '___';
 
 /**
  * Utility functions for generating collection information or parsing collection information
@@ -27,7 +28,7 @@ type CollectionInfo = {
  * @alias module:Collections
  */
 export const constructCollectionId = (name: string, version: string) =>
-  `${name}___${version}`;
+  `${name}${collectionIdSeparator}${version}`;
 
 /**
  * Returns the name and version of a collection based on
@@ -40,17 +41,19 @@ export const deconstructCollectionId = (collectionId: string) => {
   let name;
   let version;
   try {
-    [name, version] = collectionId.split('___');
+    const last = collectionId.lastIndexOf(collectionIdSeparator);
+    name = collectionId.substring(0, last);
+    version = collectionId.substring(last + collectionIdSeparator.length);
+    if (name && version) {
+      return {
+        name,
+        version,
+      };
+    }
   } catch (error) {
-    throw new Error(`invalid collectionId: ${JSON.stringify(collectionId)}`);
+    // do nothing, error thrown below
   }
-  if (name && version) {
-    return {
-      name,
-      version,
-    };
-  }
-  throw new Error(`invalid collectionId: ${collectionId}`);
+  throw new Error(`invalid collectionId: ${JSON.stringify(collectionId)}`);
 };
 
 /**

package/src/DeadLetterMessage.ts

@@ -0,0 +1,54 @@
+import { SQSRecord, EventBridgeEvent } from 'aws-lambda';
+
+import { parseSQSMessageBody } from '@cumulus/aws-client/SQS';
+import { CumulusMessage } from '@cumulus/types/message';
+import Logger from '@cumulus/logger';
+
+import { getCumulusMessageFromExecutionEvent } from './StepFunctions';
+
+const log = new Logger({ sender: '@cumulus/DeadLetterMessage' });
+
+type StepFunctionEventBridgeEvent = EventBridgeEvent<'Step Functions Execution Status Change', { [key: string]: string }>;
+type UnwrapDeadLetterCumulusMessageReturnType = (
+  StepFunctionEventBridgeEvent
+  | AWS.SQS.Message
+  | SQSRecord
+  | CumulusMessage
+);
+
+/**
+ * Unwrap dead letter Cumulus message, which may be wrapped in a
+ * States cloudwatch event, which is wrapped in an SQS message.
+ *
+ * @param {Object} messageBody - received SQS message
+ * @returns {Object} the cumulus message or nearest available object
+ */
+export const unwrapDeadLetterCumulusMessage = async (
+  messageBody: UnwrapDeadLetterCumulusMessageReturnType
+): Promise<UnwrapDeadLetterCumulusMessageReturnType> => {
+  try {
+    if ('cumulus_meta' in messageBody) {
+      return messageBody;
+    }
+    if ('Body' in messageBody || 'body' in messageBody) {
+      // AWS.SQS.Message/SQS.Record case
+      const unwrappedMessageBody = parseSQSMessageBody(
+        messageBody
+      ) as CumulusMessage;
+      return await unwrapDeadLetterCumulusMessage(unwrappedMessageBody);
+    }
+    if (!('detail' in messageBody)) {
+      // Non-typed catchall
+      return messageBody;
+    }
+    // StepFunctionEventBridgeEvent case
+    const unwrappedMessageBody = await getCumulusMessageFromExecutionEvent(messageBody);
+    return await unwrapDeadLetterCumulusMessage(unwrappedMessageBody);
+  } catch (error) {
+    log.error(
+      'Falling back to storing wrapped message after encountering unwrap error',
+      error
+    );
+    return messageBody;
+  }
+};

package/src/StepFunctions.ts

@@ -8,14 +8,41 @@
  * const StepFunctions = require('@cumulus/message/StepFunctions');
  */
 
+import { EventBridgeEvent } from 'aws-lambda';
 import { JSONPath } from 'jsonpath-plus';
+import get from 'lodash/get';
+import set from 'lodash/set';
+
+import { getExecutionHistory } from '@cumulus/aws-client/StepFunctions';
+import { getStepExitedEvent, getTaskExitedEventOutput } from '@cumulus/common/execution-history';
+import { Message } from '@cumulus/types';
 import * as s3Utils from '@cumulus/aws-client/S3';
 import Logger from '@cumulus/logger';
-import { Message } from '@cumulus/types';
+
+import { getMessageExecutionArn } from './Executions';
 
 const log = new Logger({
   sender: '@cumulus/message/StepFunctions',
 });
+type ExecutionStatus = ('ABORTED' | 'RUNNING' | 'TIMED_OUT' | 'SUCCEEDED' | 'FAILED');
+
+type ExecutionStatusToWorkflowStatusMap = {
+  [K in ExecutionStatus]: Message.WorkflowStatus;
+};
+
+const executionStatusToWorkflowStatus = (
+  executionStatus: ExecutionStatus
+): Message.WorkflowStatus => {
+  const statusMap: ExecutionStatusToWorkflowStatusMap = {
+    ABORTED: 'failed',
+    FAILED: 'failed',
+    RUNNING: 'running',
+    SUCCEEDED: 'completed',
+    TIMED_OUT: 'failed',
+  };
+
+  return statusMap[executionStatus];
+};
 
 /**
  * Given a Step Function event, replace specified key in event with contents
@@ -91,3 +118,133 @@ export const parseStepMessage = async (
   }
   return <Message.CumulusMessage>parsedMessage;
 };
+
+/**
+ * Searches the Execution step History for the TaskStateEntered pertaining to
+ * the failed task Id.  HistoryEvent ids are numbered sequentially, starting at
+ * one.
+*
+ * @param {HistoryEvent[]} events - Step Function events array
+ * @param {HistoryEvent} failedStepEvent - Step Function's failed event.
+ * @returns {string} name of the current stepfunction task or 'UnknownFailedStepName'.
+ */
+export const getFailedStepName = (
+  events: AWS.StepFunctions.HistoryEvent[],
+  failedStepEvent: { id: number }
+) => {
+  try {
+    const previousEvents = events.slice(0, failedStepEvent.id - 1);
+    const startEvents = previousEvents.filter(
+      (e) => e.type === 'TaskStateEntered'
+    );
+    const stateName = startEvents.pop()?.stateEnteredEventDetails?.name;
+    if (!stateName) throw new Error('TaskStateEntered Event Object did not have `stateEnteredEventDetails.name`');
+    return stateName;
+  } catch (error) {
+    log.info('Failed to determine a failed stepName from execution events.');
+    log.error(error);
+  }
+  return 'UnknownFailedStepName';
+};
+
+/**
+ * Finds all failed execution events and returns the last one in the list.
+ *
+ * @param {Array<HistoryEventList>} events - array of AWS Stepfunction execution HistoryEvents
+ * @returns {HistoryEventList | undefined} - the last lambda or activity that failed in the
+ * event array, or an empty array.
+ */
+export const lastFailedEventStep = (
+  events: AWS.StepFunctions.HistoryEvent[]
+): AWS.StepFunctions.HistoryEvent | undefined => {
+  const failures = events.filter((event) =>
+    ['LambdaFunctionFailed', 'ActivityFailed'].includes(event.type));
+  return failures.pop();
+};
+
+/**
+ * Get message to use for publishing failed execution notifications.
+ *
+ * Try to get the input to the last failed step in the execution so we can
+ * update the status of any granules/PDRs that don't exist in the initial execution
+ * input.
+ *
+ * Falls back to overall execution input.
+ *
+ * @param {Object} inputCumulusMessage - Workflow execution input message
+ * @param {Function} getExecutionHistoryFunction - Testing override for mock/etc of
+ *                                                 StepFunctions.getExecutionHistory
+ * @returns {Object} - CumulusMessage Execution step message or execution input message
+ */
+export const getFailedExecutionMessage = async (
+  inputCumulusMessage: Message.CumulusMessage,
+  getExecutionHistoryFunction: typeof getExecutionHistory = getExecutionHistory
+) => {
+  const amendedCumulusMessage = { ...inputCumulusMessage };
+
+  try {
+    const executionArn = getMessageExecutionArn(amendedCumulusMessage);
+    if (!executionArn) { throw new Error('No execution arn found'); }
+    const { events } = await getExecutionHistoryFunction({ executionArn });
+
+    const lastFailedEvent = lastFailedEventStep(events);
+    if (!lastFailedEvent) {
+      log.warn(`No failed step events found in execution ${executionArn}`);
+      return amendedCumulusMessage;
+    }
+    const failedExecutionStepName = getFailedStepName(events, lastFailedEvent);
+    const failedStepExitedEvent = getStepExitedEvent(events, lastFailedEvent);
+
+    if (!failedStepExitedEvent) {
+      log.info(`Could not retrieve output from last failed step in execution ${executionArn}, falling back to execution input`);
+      log.info(`Could not find TaskStateExited event after step ID ${lastFailedEvent.id} for execution ${executionArn}`);
+      return {
+        ...amendedCumulusMessage,
+        exception: {
+          ...lastFailedEvent.activityFailedEventDetails,
+          ...lastFailedEvent.lambdaFunctionFailedEventDetails,
+          failedExecutionStepName,
+
+        },
+      };
+    }
+    const taskExitedEventOutput = JSON.parse(getTaskExitedEventOutput(failedStepExitedEvent) || '{}');
+    taskExitedEventOutput.exception = {
+      ...taskExitedEventOutput.exception,
+      failedExecutionStepName,
+    };
+    return await parseStepMessage(taskExitedEventOutput, failedExecutionStepName);
+  } catch (error) {
+    log.error('getFailedExecution failed to retrieve failure:', error);
+    return amendedCumulusMessage;
+  }
+};
+
+export const getCumulusMessageFromExecutionEvent = async (executionEvent: EventBridgeEvent<'Step Functions Execution Status Change', { [key: string]: string }>) => {
+  let cumulusMessage;
+  if (executionEvent.detail.status === 'RUNNING') {
+    cumulusMessage = JSON.parse(executionEvent.detail.input);
+  } else if (executionEvent.detail.status === 'SUCCEEDED') {
+    cumulusMessage = JSON.parse(executionEvent.detail.output);
+  } else {
+    const inputMessage = JSON.parse(get(executionEvent, 'detail.input', '{}'));
+    cumulusMessage = await getFailedExecutionMessage(inputMessage);
+  }
+
+  const fullCumulusMessage = (await pullStepFunctionEvent(
+    cumulusMessage
+  )) as Message.CumulusMessage;
+
+  const workflowStatus = executionStatusToWorkflowStatus(
+    executionEvent.detail.status as ExecutionStatus
+  );
+  set(fullCumulusMessage, 'meta.status', workflowStatus);
+
+  set(
+    fullCumulusMessage,
+    'cumulus_meta.workflow_stop_time',
+    executionEvent.detail.stopDate
+  );
+
+  return fullCumulusMessage;
+};