@cumulus/message 10.1.2 → 11.1.1

package/src/Granules.ts

@@ -8,8 +8,23 @@
  * const Granules = require('@cumulus/message/Granules');
  */
 
+import isInteger from 'lodash/isInteger';
+import isNil from 'lodash/isNil';
+import mapValues from 'lodash/mapValues';
+import omitBy from 'lodash/omitBy';
+
+import { CumulusMessageError } from '@cumulus/errors';
 import { Message } from '@cumulus/types';
-import { ApiGranule, GranuleStatus } from '@cumulus/types/api/granules';
+import { ExecutionProcessingTimes } from '@cumulus/types/api/executions';
+import {
+  ApiGranule,
+  GranuleStatus,
+  GranuleTemporalInfo,
+  MessageGranule,
+} from '@cumulus/types/api/granules';
+import { ApiFile } from '@cumulus/types/api/files';
+
+import { CmrUtilsClass } from './types';
 
 interface MetaWithGranuleQueryFields extends Message.Meta {
   granule?: {
@@ -53,14 +68,14 @@ export const messageHasGranules = (
  * Determine the status of a granule.
  *
  * @param {string} workflowStatus - The workflow status
- * @param {ApiGranule} granule - A granule record
+ * @param {MessageGranule} granule - A granule record conforming to the 'api' schema
  * @returns {string} The granule status
  *
  * @alias module:Granules
  */
 export const getGranuleStatus = (
   workflowStatus: Message.WorkflowStatus,
-  granule: ApiGranule
+  granule: MessageGranule
 ): Message.WorkflowStatus | GranuleStatus => workflowStatus || granule.status;
 
 /**
@@ -74,3 +89,203 @@ export const getGranuleStatus = (
 export const getGranuleQueryFields = (
   message: MessageWithGranules
 ) => message.meta?.granule?.queryFields;
+
+/**
+ * Calculate granule product volume, which is the sum of the file
+ * sizes in bytes
+ *
+ * @param {Array<Object>} granuleFiles - array of granule file objects that conform to the
+ *                                       Cumulus 'api' schema
+ * @returns {string} - sum of granule file sizes in bytes as a string
+ */
+export const getGranuleProductVolume = (granuleFiles: ApiFile[] = []): string => {
+  if (granuleFiles.length === 0) return '0';
+  return String(granuleFiles
+    .map((f) => f.size ?? 0)
+    .filter(isInteger)
+    .reduce((x, y) => x + BigInt(y), BigInt(0)));
+};
+
+export const getGranuleTimeToPreprocess = ({
+  sync_granule_duration = 0,
+} = {}) => sync_granule_duration / 1000;
+
+export const getGranuleTimeToArchive = ({
+  post_to_cmr_duration = 0,
+} = {}) => post_to_cmr_duration / 1000;
+
+/**
+ * Convert date string to standard ISO format.
+ *
+ * @param {string} date - Date string, possibly in multiple formats
+ * @returns {string} Standardized ISO date string
+ */
+const convertDateToISOString = (date: string) => new Date(date).toISOString();
+
+function isProcessingTimeInfo(
+  info: ExecutionProcessingTimes | {} = {}
+): info is ExecutionProcessingTimes {
+  return (info as ExecutionProcessingTimes)?.processingStartDateTime !== undefined
+    && (info as ExecutionProcessingTimes)?.processingEndDateTime !== undefined;
+}
+
+/**
+ * Convert granule processing timestamps to a standardized ISO string
+ * format for compatibility across database systems.
+ *
+ * @param {ExecutionProcessingTimes} [processingTimeInfo]
+ *   Granule processing time info, if any
+ * @returns {Promise<ExecutionProcessingTimes | undefined>}
+ */
+export const getGranuleProcessingTimeInfo = (
+  processingTimeInfo?: ExecutionProcessingTimes
+): ExecutionProcessingTimes | {} => {
+  const updatedProcessingTimeInfo = isProcessingTimeInfo(processingTimeInfo)
+    ? { ...processingTimeInfo }
+    : {};
+  return mapValues(
+    updatedProcessingTimeInfo,
+    convertDateToISOString
+  );
+};
+
+function isGranuleTemporalInfo(
+  info: GranuleTemporalInfo | {} = {}
+): info is GranuleTemporalInfo {
+  return (info as GranuleTemporalInfo)?.beginningDateTime !== undefined
+    && (info as GranuleTemporalInfo)?.endingDateTime !== undefined
+    && (info as GranuleTemporalInfo)?.productionDateTime !== undefined
+    && (info as GranuleTemporalInfo)?.lastUpdateDateTime !== undefined;
+}
+
+/**
+ * Get granule temporal information from argument or directly from CMR.
+ *
+ * Converts temporal information timestamps to a standardized ISO string
+ * format for compatibility across database systems.
+ *
+ * @param {Object} params
+ * @param {MessageGranule} params.granule - Granule from workflow message
+ * @param {Object} [params.cmrTemporalInfo] - CMR temporal info, if any
+ * @param {CmrUtilsClass} params.cmrUtils - CMR utilities object
+ * @returns {Promise<GranuleTemporalInfo | undefined>}
+ */
+export const getGranuleCmrTemporalInfo = async ({
+  granule,
+  cmrTemporalInfo,
+  cmrUtils,
+}: {
+  granule: MessageGranule,
+  cmrTemporalInfo?: GranuleTemporalInfo,
+  cmrUtils: CmrUtilsClass
+}): Promise<GranuleTemporalInfo | {}> => {
+  // Get CMR temporalInfo (beginningDateTime, endingDateTime,
+  // productionDateTime, lastUpdateDateTime)
+  const temporalInfo = isGranuleTemporalInfo(cmrTemporalInfo)
+    ? { ...cmrTemporalInfo }
+    : await cmrUtils.getGranuleTemporalInfo(granule);
+  return mapValues(
+    temporalInfo,
+    convertDateToISOString
+  );
+};
+
+/**
+ * Generate an API granule record
+ *
+ * @param {MessageWithGranules} message - A workflow message
+ * @returns {Promise<ApiGranule>} The granule API record
+ *
+ * @alias module:Granules
+ */
+export const generateGranuleApiRecord = async ({
+  granule,
+  executionUrl,
+  collectionId,
+  provider,
+  workflowStartTime,
+  error,
+  pdrName,
+  status,
+  queryFields,
+  updatedAt,
+  files,
+  processingTimeInfo,
+  cmrUtils,
+  timestamp,
+  duration,
+  productVolume,
+  timeToPreprocess,
+  timeToArchive,
+  cmrTemporalInfo,
+}: {
+  granule: MessageGranule,
+  executionUrl?: string,
+  collectionId: string,
+  provider?: string,
+  workflowStartTime: number,
+  error?: Object,
+  pdrName?: string,
+  status: GranuleStatus,
+  queryFields?: Object,
+  updatedAt: number,
+  processingTimeInfo?: ExecutionProcessingTimes,
+  files?: ApiFile[],
+  timestamp: number,
+  cmrUtils: CmrUtilsClass
+  cmrTemporalInfo?: GranuleTemporalInfo,
+  duration: number,
+  productVolume: string,
+  timeToPreprocess: number,
+  timeToArchive: number,
+}): Promise<ApiGranule> => {
+  if (!granule.granuleId) throw new CumulusMessageError(`Could not create granule record, invalid granuleId: ${granule.granuleId}`);
+
+  if (!collectionId) {
+    throw new CumulusMessageError('collectionId required to generate a granule record');
+  }
+
+  const {
+    granuleId,
+    cmrLink,
+    published = false,
+    createdAt,
+  } = granule;
+
+  const now = Date.now();
+  const recordUpdatedAt = updatedAt ?? now;
+  const recordTimestamp = timestamp ?? now;
+
+  // Get CMR temporalInfo
+  const temporalInfo = await getGranuleCmrTemporalInfo({
+    granule,
+    cmrTemporalInfo,
+    cmrUtils,
+  });
+  const updatedProcessingTimeInfo = getGranuleProcessingTimeInfo(processingTimeInfo);
+
+  const record = {
+    granuleId,
+    pdrName,
+    collectionId,
+    status,
+    provider,
+    execution: executionUrl,
+    cmrLink,
+    files,
+    error,
+    published,
+    createdAt: createdAt || workflowStartTime,
+    timestamp: recordTimestamp,
+    updatedAt: recordUpdatedAt,
+    duration,
+    productVolume,
+    timeToPreprocess,
+    timeToArchive,
+    ...updatedProcessingTimeInfo,
+    ...temporalInfo,
+    queryFields,
+  };
+
+  return <ApiGranule>omitBy(record, isNil);
+};

package/src/PDRs.ts

@@ -1,4 +1,25 @@
+import {
+  CumulusMessageError,
+} from '@cumulus/errors';
+import Logger from '@cumulus/logger';
 import { Message } from '@cumulus/types';
+import { ApiPdr } from '@cumulus/types/api/pdrs';
+
+import { getCollectionIdFromMessage } from './Collections';
+import {
+  getMessageExecutionArn,
+  getExecutionUrlFromArn,
+} from './Executions';
+import {
+  getMessageProviderId,
+} from './Providers';
+import {
+  getMetaStatus,
+  getMessageWorkflowStartTime,
+  getWorkflowDuration,
+} from './workflows';
+
+const logger = new Logger({ sender: '@cumulus/message/PDRs' });
 
 interface PDR {
   name: string
@@ -9,12 +30,6 @@ interface PDR {
 interface MessageWithOptionalPayloadPdr extends Message.CumulusMessage {
   payload: {
     pdr?: PDR
-  }
-}
-
-interface MessageWithOptionalPdrStats extends Message.CumulusMessage {
-  payload: {
-    pdr: PDR
     failed?: unknown[]
     running?: unknown[]
     completed?: unknown[]
@@ -91,37 +106,37 @@ export const getMessagePdrName = (
 /**
  * Get the number of running executions for a PDR, if any.
  *
- * @param {MessageWithOptionalPdrStats} message - A workflow message
+ * @param {MessageWithOptionalPayloadPdr} message - A workflow message
  * @returns {number} Number of running executions
  *
  * @alias module:PDRs
  */
 export const getMessagePdrRunningExecutions = (
-  message: MessageWithOptionalPdrStats
+  message: MessageWithOptionalPayloadPdr
 ): number => (message.payload.running ?? []).length;
 
 /**
  * Get the number of completed executions for a PDR, if any.
  *
- * @param {MessageWithOptionalPdrStats} message - A workflow message
+ * @param {MessageWithOptionalPayloadPdr} message - A workflow message
  * @returns {number} Number of completed executions
  *
  * @alias module:PDRs
  */
 export const getMessagePdrCompletedExecutions = (
-  message: MessageWithOptionalPdrStats
+  message: MessageWithOptionalPayloadPdr
 ): number => (message.payload.completed ?? []).length;
 
 /**
  * Get the number of failed executions for a PDR, if any.
  *
- * @param {MessageWithOptionalPdrStats} message - A workflow message
+ * @param {MessageWithOptionalPayloadPdr} message - A workflow message
  * @returns {number} Number of failed executions
  *
  * @alias module:PDRs
  */
 export const getMessagePdrFailedExecutions = (
-  message: MessageWithOptionalPdrStats
+  message: MessageWithOptionalPayloadPdr
 ): number => (message.payload.failed ?? []).length;
 
 /**
@@ -134,7 +149,7 @@ export const getMessagePdrFailedExecutions = (
  * @alias module:PDRs
  */
 export const getMessagePdrStats = (
-  message: MessageWithOptionalPdrStats
+  message: MessageWithOptionalPayloadPdr
 ): PdrStats => {
   const processing = getMessagePdrRunningExecutions(message);
   const completed = getMessagePdrCompletedExecutions(message);
@@ -167,3 +182,74 @@ export const getPdrPercentCompletion = (
   }
   return progress;
 };
+
+/**
+ * Generate a PDR record for the API from the message.
+ *
+ * @param {MessageWithOptionalPayloadPdr} message - A workflow message object
+ * @param {string} [updatedAt] - Optional updated timestamp to apply to record
+ * @returns {ExecutionRecord} An PDR API record
+ *
+ * @alias module:Executions
+ */
+export const generatePdrApiRecordFromMessage = (
+  message: MessageWithOptionalPayloadPdr,
+  updatedAt = Date.now()
+): ApiPdr | undefined => {
+  const pdr = getMessagePdr(message);
+
+  // We got a message with no PDR (OK)
+  if (!pdr) {
+    logger.info('No PDRs to process on the message');
+    return undefined;
+  }
+
+  // We got a message with a PDR but no name to identify it (Not OK)
+  if (!pdr.name) {
+    throw new CumulusMessageError(`Could not find name on PDR object ${JSON.stringify(pdr)}`);
+  }
+
+  const collectionId = getCollectionIdFromMessage(message);
+  if (!collectionId) {
+    throw new CumulusMessageError('meta.collection required to generate a PDR record');
+  }
+
+  const providerId = getMessageProviderId(message);
+  if (!providerId) {
+    throw new CumulusMessageError('meta.provider required to generate a PDR record');
+  }
+
+  const status = getMetaStatus(message);
+  if (!status) {
+    throw new CumulusMessageError('meta.status required to generate a PDR record');
+  }
+
+  const arn = getMessageExecutionArn(message);
+  if (!arn) {
+    throw new CumulusMessageError('cumulus_meta.state_machine and cumulus_meta.execution_name required to generate a PDR record');
+  }
+  const execution = getExecutionUrlFromArn(arn);
+
+  const stats = getMessagePdrStats(message);
+  const progress = getPdrPercentCompletion(stats);
+  const now = Date.now();
+  const workflowStartTime = getMessageWorkflowStartTime(message);
+
+  const record = {
+    pdrName: pdr.name,
+    collectionId,
+    status,
+    provider: providerId,
+    progress,
+    execution,
+    PANSent: getMessagePdrPANSent(message),
+    PANmessage: getMessagePdrPANMessage(message),
+    stats,
+    createdAt: getMessageWorkflowStartTime(message),
+    timestamp: now,
+    updatedAt,
+    duration: getWorkflowDuration(workflowStartTime, now),
+  };
+
+  return record;
+};

package/src/types.ts

@@ -1,4 +1,5 @@
 import { Message } from '@cumulus/types';
+import { GranuleTemporalInfo, MessageGranule } from '@cumulus/types/api/granules';
 
 export interface WorkflowMessageTemplateCumulusMeta {
   queueExecutionLimits: Message.QueueExecutionLimits
@@ -15,3 +16,7 @@ export interface Workflow {
   arn: string
   name: string
 }
+
+export interface CmrUtilsClass {
+  getGranuleTemporalInfo(granule: MessageGranule): Promise<GranuleTemporalInfo | {}>
+}

package/src/utils.ts

@@ -0,0 +1,17 @@
+import isNil from 'lodash/isNil';
+import isObject from 'lodash/isObject';
+
+/**
+ * Ensures that the exception is returned as an object
+ *
+ * @param {Object|undefined} exception - the exception
+ * @returns {string} an stringified exception
+ */
+export const parseException = (exception: Object | undefined) => {
+  if (isNil(exception)) return {};
+  if (isObject(exception)) return exception;
+  return {
+    Error: 'Unknown Error',
+    Cause: exception,
+  };
+};