@monorise/core 1.0.2 → 1.0.4-0

package/processors/prejoin-processor.ts

@@ -1,264 +0,0 @@
-import type { Entity } from '@monorise/base';
-import type { SQSBatchItemFailure, SQSEvent } from 'aws-lambda';
-import { AllowedEntityTypes, EntityConfig } from '#/lambda-layer/monorise';
-import type { MutualRepository } from '../data/Mutual';
-import { PROJECTION_EXPRESSION } from '../data/ProjectionExpression';
-import { parseSQSBusEvent } from '../helpers/event';
-import type { publishEvent as publishEventType } from '../helpers/event';
-import type { EventDetailBody as MutualProcessorEventDetailBody } from '../processors/mutual-processor';
-import { DependencyContainer } from '../services/DependencyContainer';
-import type { Prejoins } from '../types/entity.type';
-import { EVENT } from '../types/event';
-
-export type EventDetailBody = {
-  byEntityType: Entity;
-  byEntityId: string;
-  entityType: Entity;
-  publishedAt: string;
-};
-
-const container = new DependencyContainer();
-
-async function processPrejoins({
-  mutualRepository,
-  publishEvent,
-  byEntityType,
-  byEntityId,
-  prejoins,
-  publishedAt,
-}: {
-  mutualRepository: MutualRepository;
-  publishEvent: typeof publishEventType;
-  byEntityType: Entity;
-  byEntityId: string;
-  prejoins: Prejoins;
-  publishedAt: string;
-}) {
-  const mutualCache: Partial<
-    Record<
-      Entity,
-      Array<{
-        byEntityType?: Entity;
-        byEntityId?: string;
-        entityType: Entity;
-        entityId: string;
-      }>
-    >
-  > = {
-    /*
-        course: [{
-            byEntityType: 'module',
-            byEntityId: '1',
-            entityType: 'course',
-            entityId: '1',
-        }],
-        module: [],
-        chapter: [],
-        video: []
-    */
-  };
-
-  //initiate
-  mutualCache[byEntityType] = [
-    {
-      entityType: byEntityType,
-      entityId: byEntityId,
-    },
-  ];
-
-  for (const { mutualField, targetEntityType, entityPaths } of prejoins) {
-    let toBePublishedContext: Record<string, any> = {};
-
-    for (const [index, entityPath] of entityPaths.entries()) {
-      const entityType = entityPath.entityType as keyof typeof mutualCache;
-
-      // skip cached
-      if (!entityPath.skipCache && mutualCache[entityType]) {
-        continue;
-      }
-
-      // if skipping cache should not have previous run data
-      mutualCache[entityType] = [];
-
-      const parentEntityType = entityPaths[index - 1]
-        .entityType as keyof typeof mutualCache;
-      const parentEntities = mutualCache[parentEntityType] ?? [];
-
-      // find all nested entities
-      for (const parentEntity of parentEntities) {
-        const { entityType: parentEntityType, entityId: parentEntityId } =
-          parentEntity;
-
-        const { items: mutualItems } =
-          await mutualRepository.listEntitiesByEntity(
-            parentEntityType,
-            parentEntityId,
-            entityPath.entityType,
-            {
-              ProjectionExpression: PROJECTION_EXPRESSION.MUTUAL_DATA_ONLY,
-            },
-          );
-
-        // custom processor defined in prejoin config for each path
-        const processor =
-          entityPath.processor || ((items, context) => ({ items, context }));
-
-        const processed = processor(mutualItems, toBePublishedContext);
-        toBePublishedContext = processed?.context || toBePublishedContext;
-
-        mutualCache[entityType] = [
-          ...(mutualCache[entityType] ?? []),
-          ...(processed ? processed.items : mutualItems),
-        ];
-      }
-
-      if (!mutualCache[entityType] && !Array.isArray(mutualCache[entityType])) {
-        // to avoid empty array
-        mutualCache[entityType] = [];
-      }
-    }
-
-    const mutualIds = (
-      mutualCache[targetEntityType as keyof typeof mutualCache] ?? []
-    ).map((item) => item.entityId);
-
-    await publishEvent<MutualProcessorEventDetailBody>({
-      event: EVENT.CORE.ENTITY_MUTUAL_TO_UPDATE,
-      payload: {
-        byEntityType,
-        byEntityId,
-        entityType: targetEntityType,
-        field: mutualField,
-        mutualIds,
-        customContext: toBePublishedContext,
-        publishedAt,
-      },
-    });
-  }
-}
-
-async function publishToSubscribers({
-  mutualRepository,
-  publishEvent,
-  byEntityType,
-  byEntityId,
-  publishedAt,
-}: {
-  mutualRepository: MutualRepository;
-  publishEvent: typeof publishEventType;
-  byEntityType: Entity;
-  byEntityId: string;
-  publishedAt: string;
-}) {
-  const listeners = AllowedEntityTypes.reduce(
-    (acc, configKey) => {
-      const { subscribes } = EntityConfig[configKey].mutual ?? {};
-
-      const hasSubscription = (subscribes ?? []).some(
-        ({ entityType }) => entityType === byEntityType,
-      );
-
-      return [
-        ...acc,
-        ...(hasSubscription
-          ? [
-              {
-                entityType: configKey,
-              },
-            ]
-          : []),
-      ];
-    },
-    [] as { entityType: Entity }[],
-  );
-
-  // publish event for each interested entity
-  const subscribedMutualItems = await Promise.all(
-    listeners.map(({ entityType: subscribedEntityType }) =>
-      mutualRepository.listEntitiesByEntity(
-        byEntityType,
-        byEntityId,
-        subscribedEntityType,
-        {
-          ProjectionExpression: PROJECTION_EXPRESSION.NO_DATA,
-        },
-      ),
-    ),
-  );
-  const subscribedMutuals = subscribedMutualItems.flatMap((item) => item.items);
-
-  await Promise.all(
-    subscribedMutuals.map((subscribedMutual) =>
-      publishEvent({
-        event: EVENT.CORE.PREJOIN_RELATIONSHIP_SYNC,
-        payload: {
-          byEntityType: subscribedMutual.entityType,
-          byEntityId: subscribedMutual.entityId,
-          entityType: subscribedMutual.byEntityType,
-          publishedAt,
-        },
-      }),
-    ),
-  );
-}
-
-export const handler = async (ev: SQSEvent) => {
-  const batchItemFailures: SQSBatchItemFailure[] = [];
-
-  const { mutualRepository, publishEvent } = container;
-
-  for (const record of ev.Records) {
-    const body = parseSQSBusEvent<EventDetailBody>(record.body);
-    const { detail } = body;
-    const { byEntityType, byEntityId, entityType, publishedAt } = detail;
-    let errorContext: Record<string, unknown> = {
-      body,
-    };
-
-    try {
-      const isEntityTypeSubscribed = (
-        EntityConfig[byEntityType]?.mutual?.subscribes ?? []
-      ).some(
-        ({ entityType: subscribedEntityType }) =>
-          subscribedEntityType === entityType,
-      );
-      const hasPrejoins = EntityConfig[byEntityType]?.mutual?.prejoins;
-      const shouldProcessPrejoins = isEntityTypeSubscribed && hasPrejoins;
-      errorContext = {
-        ...errorContext,
-        isEntityTypeSubscribed,
-        hasPrejoins,
-        shouldProcessPrejoins,
-      };
-
-      if (shouldProcessPrejoins) {
-        await processPrejoins({
-          mutualRepository,
-          publishEvent,
-          byEntityType,
-          byEntityId,
-          prejoins: EntityConfig[byEntityType]?.mutual?.prejoins ?? [],
-          publishedAt,
-        });
-      }
-
-      await publishToSubscribers({
-        mutualRepository,
-        publishEvent,
-        byEntityType,
-        byEntityId,
-        publishedAt,
-      });
-    } catch (err) {
-      console.log(
-        '===PREJOIN-PROCESSOR ERROR===',
-        err,
-        JSON.stringify({ errorContext }, null, 2),
-      );
-
-      batchItemFailures.push({ itemIdentifier: record.messageId });
-    }
-  }
-
-  return { batchItemFailures };
-};

package/processors/replication-processor.ts

@@ -1,261 +0,0 @@
-import { ConditionalCheckFailedException } from '@aws-sdk/client-dynamodb';
-import type {
-  AttributeValue,
-  _Record as DynamoDBStreamEvent,
-} from '@aws-sdk/client-dynamodb-streams';
-import type { DynamoDBBatchItemFailure } from 'aws-lambda';
-import {
-  ENTITY_REPLICATION_INDEX,
-  MUTUAL_REPLICATION_INDEX,
-} from '../configs/service.config';
-import { StandardError } from '../errors/standard-error';
-import { DependencyContainer } from '../services/DependencyContainer';
-
-const container = new DependencyContainer();
-const TableName = process.env.DDB_TABLE;
-
-export const handler = async (event: { Records: DynamoDBStreamEvent[] }) => {
-  const batchItemFailures: DynamoDBBatchItemFailure[] = [];
-  const { dynamodbClient } = container;
-
-  for (const record of event.Records) {
-    const errorContext: any = {};
-    errorContext.record = record;
-
-    try {
-      if (record.eventName === 'MODIFY') {
-        const modifiedItem = record.dynamodb?.NewImage;
-        if (!modifiedItem) {
-          continue;
-        }
-
-        const isMetadata = modifiedItem.SK.S?.startsWith('#METADATA#');
-        const isMutual = modifiedItem.PK.S?.startsWith('MUTUAL#') && isMetadata;
-        const isEntity = isMetadata && !isMutual;
-
-        if (!isEntity && !isMutual) {
-          // skip replicated data
-          continue;
-        }
-
-        // default variables
-        let targetRPK = 'R1PK';
-        const targetData = 'data';
-        let targetIndexName = ENTITY_REPLICATION_INDEX;
-        const targetUpdatedAt = 'updatedAt';
-
-        let queryExpression: {
-          FilterExpression?: string;
-          ExpressionAttributeNames: Record<string, string>;
-          ExpressionAttributeValues: Record<string, AttributeValue>;
-        } = {
-          FilterExpression: `#${targetUpdatedAt} < :${targetUpdatedAt}`,
-          ExpressionAttributeNames: {
-            [`#${targetRPK}`]: targetRPK,
-            [`#${targetUpdatedAt}`]: targetUpdatedAt,
-          },
-          ExpressionAttributeValues: {
-            [`:${targetRPK}`]: modifiedItem.PK,
-            [`:${targetUpdatedAt}`]: modifiedItem.updatedAt,
-          },
-        };
-
-        const updateExpession: {
-          ConditionExpression: string;
-          UpdateExpression: string;
-          ExpressionAttributeNames: Record<string, string>;
-          ExpressionAttributeValues: Record<string, AttributeValue>;
-        } = {
-          UpdateExpression: `SET #${targetUpdatedAt} = :${targetUpdatedAt}, #${targetData} = :${targetData}`,
-          ConditionExpression: `#${targetUpdatedAt} < :${targetUpdatedAt}`,
-          ExpressionAttributeNames: {
-            [`#${targetData}`]: targetData,
-            [`#${targetUpdatedAt}`]: targetUpdatedAt,
-          },
-          ExpressionAttributeValues: {
-            [`:${targetData}`]: modifiedItem.data,
-            [`:${targetUpdatedAt}`]: modifiedItem.updatedAt,
-          },
-        };
-
-        if (isMutual) {
-          targetRPK = 'R2PK';
-          targetIndexName = MUTUAL_REPLICATION_INDEX;
-
-          // condition to only replicate to mutualAsEntity
-          queryExpression = {
-            FilterExpression: `${queryExpression.FilterExpression} AND #SK = :metadata`, // to replicate to mutualAsEntity only
-            ExpressionAttributeNames: {
-              '#SK': 'SK',
-              [`#${targetRPK}`]: targetRPK,
-              [`#${targetUpdatedAt}`]: targetUpdatedAt,
-            },
-            ExpressionAttributeValues: {
-              ':metadata': { S: '#METADATA#' },
-              [`:${targetRPK}`]: modifiedItem.PK,
-              [`:${targetUpdatedAt}`]: modifiedItem.mutualUpdatedAt,
-            },
-          };
-
-          updateExpession.ExpressionAttributeValues = {
-            [`:${targetData}`]: modifiedItem.mutualData,
-            [`:${targetUpdatedAt}`]: modifiedItem.mutualUpdatedAt,
-          };
-        }
-
-        errorContext.queryExpression = queryExpression;
-        errorContext.updateExpession = updateExpession;
-
-        // retrieve all to be replicated items
-        let toBeReplicatedItems: Record<string, AttributeValue>[] = [];
-        let lastKey;
-
-        do {
-          const queryResult = await dynamodbClient.query({
-            TableName,
-            IndexName: targetIndexName,
-            KeyConditionExpression: `#${targetRPK} = :${targetRPK}`,
-            ...queryExpression,
-          });
-
-          toBeReplicatedItems = [
-            ...toBeReplicatedItems,
-            ...(queryResult.Items || []),
-          ];
-          lastKey = queryResult.LastEvaluatedKey;
-        } while (lastKey);
-        errorContext.toBeReplicatedItems = toBeReplicatedItems;
-
-        const updatePromises = toBeReplicatedItems.map((item) => {
-          const updateParams = {
-            TableName,
-            Key: {
-              PK: item.PK,
-              SK: item.SK,
-            },
-            ...updateExpession,
-          };
-
-          return dynamodbClient.updateItem(updateParams);
-        });
-
-        const results = await Promise.allSettled(updatePromises);
-        errorContext.results = results;
-
-        if (
-          results.some(
-            (result) =>
-              result.status === 'rejected' &&
-              !(result.reason instanceof ConditionalCheckFailedException),
-          )
-        ) {
-          throw new StandardError(
-            'REPLICATION_ERROR',
-            'Replication error',
-            null,
-            errorContext,
-          );
-        }
-      }
-
-      if (record.eventName === 'REMOVE') {
-        const removedKeys = record.dynamodb?.Keys || {};
-        const isMetadata = removedKeys.SK.S?.startsWith('#METADATA#');
-        const isMutual = removedKeys.PK.S?.startsWith('MUTUAL#') && isMetadata;
-        const isEntity = isMetadata && !isMutual;
-
-        if (!isEntity && !isMutual) {
-          continue;
-        }
-
-        // default query settings
-        let targetRPK = 'R1PK';
-        let targetIndexName: string = ENTITY_REPLICATION_INDEX;
-
-        if (isMutual) {
-          targetRPK = 'R2PK';
-          targetIndexName = MUTUAL_REPLICATION_INDEX;
-        }
-
-        let itemsToDelete: Record<string, AttributeValue>[] = [];
-        let lastKey;
-
-        do {
-          const queryResult = await dynamodbClient.query({
-            TableName,
-            IndexName: targetIndexName,
-            KeyConditionExpression: `#${targetRPK} = :${targetRPK}`,
-            ExpressionAttributeNames: {
-              [`#${targetRPK}`]: targetRPK,
-            },
-            ExpressionAttributeValues: {
-              [`:${targetRPK}`]: removedKeys.PK,
-            },
-          });
-
-          itemsToDelete = [...itemsToDelete, ...(queryResult.Items || [])];
-          lastKey = queryResult.LastEvaluatedKey;
-        } while (lastKey);
-
-        const mutualsToDelete = Array.from(
-          new Set(
-            itemsToDelete
-              .filter((item) => item.R2PK?.S?.startsWith('MUTUAL#'))
-              .map((filteredItem) => filteredItem.R2PK?.S),
-          ),
-        );
-        mutualsToDelete.forEach((mutual) => {
-          if (!mutual) {
-            return;
-          }
-
-          itemsToDelete.push({ PK: { S: mutual }, SK: { S: '#METADATA#' } });
-        });
-
-        const deleteResults = await Promise.allSettled(
-          itemsToDelete.map((item) =>
-            dynamodbClient.deleteItem({
-              TableName,
-              Key: {
-                PK: item.PK,
-                SK: item.SK,
-              },
-            }),
-          ),
-        );
-        errorContext.deleteResults = deleteResults;
-
-        if (
-          deleteResults.some(
-            (result) =>
-              result.status === 'rejected' &&
-              !(result.reason instanceof ConditionalCheckFailedException),
-          )
-        ) {
-          throw new StandardError(
-            'REPLICATION_ERROR',
-            'Replication error',
-            null,
-            errorContext,
-          );
-        }
-      }
-    } catch (error) {
-      console.error('====REPLICATION_ERROR', error);
-      console.log(
-        '====REPLICATION_ERROR errorContext',
-        JSON.stringify(errorContext, null, 2),
-      );
-
-      batchItemFailures.push({
-        itemIdentifier: record.dynamodb?.SequenceNumber || '',
-      });
-
-      // immediately return to prevent processing the rest
-      // because stream will restart from this point again
-      return { batchItemFailures };
-    }
-  }
-
-  return { batchItemFailures };
-};

package/processors/tag-processor.ts

@@ -1,174 +0,0 @@
-import type { CreatedEntity, Entity as EntityType } from '@monorise/base';
-import type { SQSBatchItemFailure, SQSEvent } from 'aws-lambda';
-import { EntityConfig } from '#/lambda-layer/monorise';
-import type { Entity } from '../data/Entity';
-import { parseSQSBusEvent } from '../helpers/event';
-import { DependencyContainer } from '../services/DependencyContainer';
-import type { Tag } from '../types/entity.type';
-
-export type EventDetailBody = {
-  entityType: EntityType;
-  entityId: string;
-  data: Record<string, any>;
-};
-
-const container = new DependencyContainer();
-
-function compareTags(
-  existingTags: Tag[],
-  newTags: Tag[],
-): {
-  old: Tag[];
-  new: Tag[];
-  remain: Tag[];
-} {
-  const oldMap = new Map(
-    existingTags.map((item) => [`${item.group}#${item.sortValue}`, item]),
-  );
-  const newMap = new Map(
-    newTags.map((item) => [`${item.group}#${item.sortValue}`, item]),
-  );
-
-  const remain: Tag[] = [];
-  const oldDiff: Tag[] = [];
-  const newDiff: Tag[] = [];
-
-  for (const [key, oldItem] of oldMap.entries()) {
-    if (newMap.has(key)) {
-      remain.push(oldItem);
-      newMap.delete(key); // Remove from newMap as it's already in remain
-    } else {
-      oldDiff.push(oldItem);
-    }
-  }
-
-  // Remaining entries in newMap are new
-  for (const newItem of newMap.values()) {
-    newDiff.push(newItem);
-  }
-
-  return {
-    old: oldDiff,
-    new: newDiff,
-    remain,
-  };
-}
-
-async function batchUpdateTags({
-  tagName,
-  entity,
-  diff,
-}: {
-  tagName: string;
-  entity: Entity<EntityType>;
-  diff: { old: Tag[]; new: Tag[] };
-}): Promise<void> {
-  if (!entity.entityId) {
-    throw new Error('entityId is required');
-  }
-
-  const { old: tagsToRemove, new: tagsToAdd } = diff;
-  const { entityType, entityId } = entity;
-
-  const removePromises = tagsToRemove.reduce(
-    (acc, tag) => [
-      ...acc,
-      container.tagRepository.deleteTag({
-        tagName,
-        group: tag.group,
-        sortValue: tag.sortValue,
-        entityType,
-        entityId,
-      }),
-    ],
-    [] as Promise<any>[],
-  );
-
-  const addPromises = tagsToAdd.reduce(
-    (acc, tag) => [
-      ...acc,
-      container.tagRepository.createTag({
-        tagName,
-        group: tag.group,
-        sortValue: tag.sortValue,
-        entity,
-      }),
-    ],
-    [] as Promise<any>[],
-  );
-
-  await Promise.all([...removePromises, ...addPromises]);
-}
-
-export const handler = async (ev: SQSEvent) => {
-  const batchItemFailures: SQSBatchItemFailure[] = [];
-
-  for (const record of ev.Records) {
-    const body = parseSQSBusEvent<EventDetailBody>(record.body);
-    const { detail } = body;
-    const { entityType, entityId } = detail;
-
-    const errorContext: Record<string, unknown> = {};
-    errorContext.body = body;
-
-    try {
-      const tagConfigs = EntityConfig[entityType]?.tags;
-
-      if (!tagConfigs || !tagConfigs.length) {
-        // skip if entity has no tag configs
-        continue;
-      }
-
-      await container.tagRepository.createLock({
-        entityType,
-        entityId,
-      });
-
-      for (const tagConfig of tagConfigs) {
-        const { name, processor } = tagConfig;
-
-        const existingTags = await container.tagRepository.getExistingTags({
-          entityType,
-          entityId,
-          tagName: name,
-        });
-        errorContext.existingTags = existingTags;
-
-        const entity = await container.entityRepository.getEntity(
-          entityType,
-          entityId,
-        );
-        errorContext.entity = entity;
-
-        const newTags = await processor(
-          entity as unknown as CreatedEntity<EntityType>,
-        );
-        errorContext.newTags = newTags;
-
-        const diff = compareTags(existingTags, newTags);
-        errorContext.diff = diff;
-
-        await batchUpdateTags({
-          tagName: name,
-          entity,
-          diff,
-        });
-      }
-
-      await container.tagRepository.deleteLock({
-        entityType,
-        entityId,
-      });
-    } catch (err) {
-      console.log(
-        '===TAG-PROCESSOR ERROR===',
-        err,
-        JSON.stringify({ errorContext }, null, 2),
-      );
-
-      batchItemFailures.push({ itemIdentifier: record.messageId });
-    }
-  }
-
-  return { batchItemFailures };
-};