@cumulus/message 18.2.1 → 18.3.0

package/src/CumulusMessage.ts

@@ -0,0 +1,9 @@
+import { Message } from '@cumulus/types';
+
+/**
+ * Bare check for CumulusMessage Shape
+ */
+export const isCumulusMessageLike = (message: any): message is Message.CumulusMessage => (
+  message instanceof Object
+  && 'cumulus_meta' in message
+);

package/src/DeadLetterMessage.ts

@@ -1,54 +1,197 @@
-import { SQSRecord, EventBridgeEvent } from 'aws-lambda';
+//@ts-check
+import { SQSRecord } from 'aws-lambda';
+import moment from 'moment';
 
-import { parseSQSMessageBody } from '@cumulus/aws-client/SQS';
+import { uuid } from 'uuidv4';
+import { isEventBridgeEvent, StepFunctionEventBridgeEvent } from '@cumulus/aws-client/Lambda';
+import { parseSQSMessageBody, isSQSRecordLike } from '@cumulus/aws-client/SQS';
 import { CumulusMessage } from '@cumulus/types/message';
+import { DLQRecord, DLARecord } from '@cumulus/types/api/dead_letters';
 import Logger from '@cumulus/logger';
-
+import { MessageGranule } from '@cumulus/types';
+import { isMessageWithProvider, getMessageProviderId } from './Providers';
+import { isCumulusMessageLike } from './CumulusMessage';
 import { getCumulusMessageFromExecutionEvent } from './StepFunctions';
+import { constructCollectionId } from './Collections';
 
 const log = new Logger({ sender: '@cumulus/DeadLetterMessage' });
 
-type StepFunctionEventBridgeEvent = EventBridgeEvent<'Step Functions Execution Status Change', { [key: string]: string }>;
-type UnwrapDeadLetterCumulusMessageReturnType = (
+type UnwrapDeadLetterCumulusMessageInputType = (
   StepFunctionEventBridgeEvent
-  | AWS.SQS.Message
-  | SQSRecord
+  | DLQRecord | SQSRecord
   | CumulusMessage
 );
 
+/**
+ * Bare check for SQS message Shape
+ */
+export const isDLQRecordLike = (message: Object): message is DLQRecord => (
+  isSQSRecordLike(message)
+  && 'error' in message
+);
+
 /**
  * Unwrap dead letter Cumulus message, which may be wrapped in a
  * States cloudwatch event, which is wrapped in an SQS message.
- *
- * @param {Object} messageBody - received SQS message
- * @returns {Object} the cumulus message or nearest available object
  */
 export const unwrapDeadLetterCumulusMessage = async (
-  messageBody: UnwrapDeadLetterCumulusMessageReturnType
-): Promise<UnwrapDeadLetterCumulusMessageReturnType> => {
+  messageBody: UnwrapDeadLetterCumulusMessageInputType
+): Promise<UnwrapDeadLetterCumulusMessageInputType> => {
   try {
-    if ('cumulus_meta' in messageBody) {
-      return messageBody;
-    }
-    if ('Body' in messageBody || 'body' in messageBody) {
+    if (isSQSRecordLike(messageBody)) {
       // AWS.SQS.Message/SQS.Record case
       const unwrappedMessageBody = parseSQSMessageBody(
         messageBody
-      ) as CumulusMessage;
+      );
       return await unwrapDeadLetterCumulusMessage(unwrappedMessageBody);
     }
-    if (!('detail' in messageBody)) {
-      // Non-typed catchall
+    if (isEventBridgeEvent(messageBody)) {
+      return await getCumulusMessageFromExecutionEvent(
+        messageBody
+      );
+    }
+    if (isCumulusMessageLike(messageBody)) {
       return messageBody;
     }
-    // StepFunctionEventBridgeEvent case
-    const unwrappedMessageBody = await getCumulusMessageFromExecutionEvent(messageBody);
-    return await unwrapDeadLetterCumulusMessage(unwrappedMessageBody);
+    throw new TypeError('DeadLetter CumulusMessage in unrecognized format');
   } catch (error) {
     log.error(
       'Falling back to storing wrapped message after encountering unwrap error',
-      error
+      error,
+      JSON.stringify(messageBody)
     );
     return messageBody;
   }
 };
+
+interface PayloadWithGranules {
+  granules: Array<MessageGranule>
+}
+
+const payloadHasGranules = (payload: any): payload is PayloadWithGranules => (
+  payload instanceof Object
+  && 'granules' in payload
+  && Array.isArray(payload.granules)
+);
+
+const extractCollectionId = (message: CumulusMessage): string | null => {
+  const collectionName = message?.meta?.collection?.name || null;
+  const collectionVersion = message?.meta?.collection?.version || null;
+  if (collectionName && collectionVersion) {
+    return constructCollectionId(collectionName, collectionVersion);
+  }
+  return null;
+};
+
+const extractGranules = (message: CumulusMessage): Array<string | null> | null => {
+  if (payloadHasGranules(message.payload)) {
+    return message.payload.granules.map((granule) => granule?.granuleId || null);
+  }
+  return null;
+};
+
+type DLQMetadata = Partial<DLQRecord> & { body: undefined };
+/**
+ * peel out metadata from an SQS(/DLQ)record
+ * @param message DLQ or SQS message
+ * @returns the given message without its body
+ */
+const extractSQSMetadata = (message: DLQRecord | SQSRecord): DLQMetadata => {
+  const metadata = { ...message } as any;
+  delete metadata.body;
+  delete metadata.Body;
+  return metadata;
+};
+
+/**
+ * Reformat object with key attributes at top level.
+ *
+ */
+export const hoistCumulusMessageDetails = async (dlqRecord: SQSRecord): Promise<DLARecord> => {
+  let executionArn = null;
+  let stateMachineArn = null;
+  let status = null;
+  let time = null;
+  let collectionId = null;
+  let granules = null;
+  let providerId = null;
+
+  let messageBody;
+  messageBody = dlqRecord;
+  let metadata = extractSQSMetadata(messageBody);
+  /* de-nest sqs records of unknown depth */
+  while (isSQSRecordLike(messageBody)) {
+    /* prefer outermost recorded metadata */
+    metadata = { ...extractSQSMetadata(messageBody), ...metadata };
+    messageBody = parseSQSMessageBody(messageBody);
+  }
+  const error = 'error' in metadata ? metadata.error : null;
+  if (isEventBridgeEvent(messageBody)) {
+    executionArn = messageBody?.detail?.executionArn || null;
+    stateMachineArn = messageBody?.detail?.stateMachineArn || null;
+    status = messageBody?.detail?.status || null;
+    time = messageBody?.time || null;
+    let cumulusMessage;
+    try {
+      cumulusMessage = await getCumulusMessageFromExecutionEvent(messageBody);
+    } catch (error_) {
+      cumulusMessage = undefined;
+      log.error(
+        'could not parse details from DLQ message body',
+        error_,
+        messageBody
+      );
+    }
+    if (cumulusMessage) {
+      collectionId = extractCollectionId(cumulusMessage);
+      granules = extractGranules(cumulusMessage);
+      if (isMessageWithProvider(cumulusMessage)) {
+        providerId = getMessageProviderId(cumulusMessage) || null;
+      }
+    }
+  } else {
+    log.error(
+      'could not parse details from DLQ message body',
+      messageBody,
+      'expected EventBridgeEvent'
+    );
+  }
+  return {
+    ...metadata,
+    body: JSON.stringify(messageBody),
+    collectionId,
+    providerId,
+    granules,
+    executionArn,
+    stateMachineArn,
+    status,
+    time,
+    error,
+  } as DLARecord; // cast to DLARecord: ts is confused by explicit 'undefined' fields in metadata
+};
+
+export const getDLARootKey = (stackName: string) => (
+  `${stackName}/dead-letter-archive/sqs/`
+);
+
+export const extractDateString = (message: DLARecord): string => (
+  message.time && moment.utc(message.time).isValid() ? moment.utc(message.time).format('YYYY-MM-DD') : moment.utc().format('YYYY-MM-DD')
+);
+
+export const extractFileName = (message: DLARecord): string => {
+  // get token after the last / or :
+  const executionName = message.executionArn ? message.executionArn.split(/[/:]/).pop() : 'unknown';
+  return `${executionName}-${uuid()}`;
+};
+
+export const getDLAKey = (stackName: string, message: DLARecord): string => {
+  const dateString = extractDateString(message);
+  const fileName = extractFileName(message);
+  return `${getDLARootKey(stackName)}${dateString}/${fileName}`;
+};
+
+export const getDLAFailureKey = (stackName: string, message: DLARecord): string => {
+  const dateString = extractDateString(message);
+  const fileName = extractFileName(message);
+  return `${stackName}/dead-letter-archive/failed-sqs/${dateString}/${fileName}`;
+};

package/src/Providers.ts

@@ -1,5 +1,5 @@
 import { Message } from '@cumulus/types';
-
+import { isCumulusMessageLike } from './CumulusMessage';
 type MessageProvider = {
   id: string
   protocol: string
@@ -7,12 +7,29 @@ type MessageProvider = {
   port?: number
 };
 
+export const isMessageProvider = (
+  obj: any
+): obj is MessageProvider => (
+  obj instanceof Object
+  // in testing instanceof String can return false for string literals
+  && 'id' in obj && (typeof (obj.id) === 'string' || obj.id instanceof String)
+  && 'protocol' in obj && (typeof (obj.protocol) === 'string' || obj.protocol instanceof String)
+  && 'host' in obj && (typeof (obj.host) === 'string' || obj.host instanceof String)
+  && ('port' in obj ? (typeof (obj.port) === 'number' || obj.port instanceof Number) : true)
+);
+
 type MessageWithProvider = Message.CumulusMessage & {
   meta: {
     provider?: MessageProvider
   }
 };
 
+export const isMessageWithProvider = (
+  obj: any
+): obj is MessageWithProvider => (
+  isCumulusMessageLike(obj)
+  && isMessageProvider(obj?.meta?.provider)
+);
 /**
  * Get the provider from a workflow message, if any.
  *

package/src/StepFunctions.ts

@@ -2,10 +2,10 @@
 
 /**
  * Utility functions for working with AWS Step Function events/messages
- * @module StepFunctions
  *
+ * @module StepFunctions
  * @example
- * const StepFunctions = require('@cumulus/message/StepFunctions');
+ *   const StepFunctions = require('@cumulus/message/StepFunctions');
  */
 
 import { EventBridgeEvent } from 'aws-lambda';
@@ -13,7 +13,7 @@ import { JSONPath } from 'jsonpath-plus';
 import get from 'lodash/get';
 import set from 'lodash/set';
 
-import { getExecutionHistory } from '@cumulus/aws-client/StepFunctions';
+import { getExecutionHistory, HistoryEvent } from '@cumulus/aws-client/StepFunctions';
 import { getStepExitedEvent, getTaskExitedEventOutput } from '@cumulus/common/execution-history';
 import { Message } from '@cumulus/types';
 import * as s3Utils from '@cumulus/aws-client/S3';
@@ -49,9 +49,9 @@ const executionStatusToWorkflowStatus = (
  * of S3 remote message
  *
  * @param {Message.CumulusRemoteMessage} event - Source event
- * @returns {Promise<Object>} Updated event with target path replaced by remote message
+ * @param event.replace - Cumulus message replace config
+ * @returns {Promise<object>} Updated event with target path replaced by remote message
  * @throws {Error} if target path cannot be found on source event
- *
  * @async
  * @alias module:StepFunctions
  */
@@ -92,8 +92,7 @@ export const pullStepFunctionEvent = async (
  *
  * @param {CMAMessage} stepMessage - Message for the step
  * @param {string} stepName - Name of the step
- * @returns {Promise<Object>} Parsed and updated event with target path replaced by remote message
- *
+ * @returns {Promise<object>} Parsed and updated event with target path replaced by remote message
  * @async
  * @alias module:StepFunctions
  */
@@ -123,17 +122,18 @@ export const parseStepMessage = async (
  * Searches the Execution step History for the TaskStateEntered pertaining to
  * the failed task Id.  HistoryEvent ids are numbered sequentially, starting at
  * one.
-*
+ *
  * @param {HistoryEvent[]} events - Step Function events array
- * @param {HistoryEvent} failedStepEvent - Step Function's failed event.
+ * @param {failedStepEvent} failedStepEvent - Step Function's failed event.
+ * @param failedStepEvent.id - number (long), Step Functions failed event id.
  * @returns {string} name of the current stepfunction task or 'UnknownFailedStepName'.
  */
 export const getFailedStepName = (
-  events: AWS.StepFunctions.HistoryEvent[],
-  failedStepEvent: { id: number }
+  events: HistoryEvent[],
+  failedStepEvent: HistoryEvent
 ) => {
   try {
-    const previousEvents = events.slice(0, failedStepEvent.id - 1);
+    const previousEvents = events.slice(0, failedStepEvent.id as number - 1);
     const startEvents = previousEvents.filter(
       (e) => e.type === 'TaskStateEntered'
     );
@@ -150,15 +150,15 @@ export const getFailedStepName = (
 /**
  * Finds all failed execution events and returns the last one in the list.
  *
- * @param {Array<HistoryEventList>} events - array of AWS Stepfunction execution HistoryEvents
- * @returns {HistoryEventList | undefined} - the last lambda or activity that failed in the
+ * @param {HistoryEvent[]} events - array of AWS Stepfunction execution HistoryEvents
+ * @returns {HistoryEvent[] | undefined} - the last lambda or activity that failed in the
  * event array, or an empty array.
  */
 export const lastFailedEventStep = (
-  events: AWS.StepFunctions.HistoryEvent[]
-): AWS.StepFunctions.HistoryEvent | undefined => {
+  events: HistoryEvent[]
+): HistoryEvent | undefined => {
   const failures = events.filter((event) =>
-    ['LambdaFunctionFailed', 'ActivityFailed'].includes(event.type));
+    ['LambdaFunctionFailed', 'ActivityFailed'].includes(event.type as string));
   return failures.pop();
 };