@unchainedshop/core 4.0.0-rc.2 → 4.0.0-rc.21

package/src/bulk-importer/createBulkImporter.ts

@@ -0,0 +1,205 @@
+import { z } from 'zod';
+import JSONStream from 'minipass-json-stream';
+import { PassThrough, Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { mongodb } from '@unchainedshop/mongodb';
+import * as AssortmentHandlers from './handlers/assortment/index.js';
+import * as FilterHandlers from './handlers/filter/index.js';
+import * as ProductHandlers from './handlers/product/index.js';
+import { createLogger } from '@unchainedshop/logger';
+import { Modules } from '../modules.js';
+import { Services } from '../services/index.js';
+
+const logger = createLogger('unchained:bulk-import');
+
+export interface BulkImportOperationResult {
+  entity: string;
+  operation: string;
+  success: boolean;
+}
+export type BulkImportOperation<T> = { payloadSchema?: z.AnyZodObject } & ((
+  payload: any,
+  options: {
+    bulk: (collection: string) => typeof mongodb.BulkOperationBase;
+    createShouldUpsertIfIDExists?: boolean;
+    updateShouldUpsertIfIDNotExists?: boolean;
+    skipCacheInvalidation?: boolean;
+  },
+  unchainedAPI: T,
+) => Promise<BulkImportOperationResult>);
+
+export type BulkImportHandler<T> = Record<string, BulkImportOperation<T>>;
+
+let bulkOperationHandlers: Record<string, BulkImportHandler<any>> = {};
+
+export const getOperation = (entity: string, operation: string): BulkImportOperation<any> => {
+  if (
+    entity === '__PROTO__' ||
+    entity === 'CONSTRUCTOR' ||
+    entity === 'PROTOTYPE' ||
+    operation === '__PROTO__' ||
+    operation === 'CONSTRUCTOR' ||
+    operation === 'PROTOTYPE'
+  ) {
+    throw new Error(`Dude!`);
+  }
+  const handlers = bulkOperationHandlers[entity];
+  if (!handlers) {
+    throw new Error(`Entity unknown`);
+  }
+
+  const entityOperation = handlers[operation] as BulkImportOperation<any>;
+
+  if (!entityOperation || typeof entityOperation !== 'function') {
+    throw new Error(`Operation unknown/invalid`);
+  }
+
+  return entityOperation;
+};
+
+export default function createBulkImporterFactory(db, bulkImporterOptions: any) {
+  bulkOperationHandlers = {
+    ASSORTMENT: AssortmentHandlers,
+    PRODUCT: ProductHandlers,
+    FILTER: FilterHandlers,
+    ...(bulkImporterOptions?.handlers || {}),
+  };
+
+  const createBulkImporter = (options) => {
+    const bulkOperations = {};
+    const preparationIssues: {
+      operation: string;
+      entity: string;
+      payloadId: string;
+      errorCode: string;
+      errorMessage: string;
+    }[] = [];
+    const processedOperations = {};
+    const { createShouldUpsertIfIDExists, skipCacheInvalidation, updateShouldUpsertIfIDNotExists } =
+      options;
+
+    const bulk = (collectionName: string) => {
+      const Collection = db.collection(collectionName);
+      if (!bulkOperations[collectionName])
+        bulkOperations[collectionName] = Collection.initializeOrderedBulkOp();
+      return bulkOperations[collectionName];
+    };
+
+    logger.debug(
+      `Configure event import with options: createShouldUpsertIfIDExists=${createShouldUpsertIfIDExists} updateShouldUpsertIfIDNotExists=${updateShouldUpsertIfIDNotExists} skipCacheInvalidation=${skipCacheInvalidation}`,
+    );
+
+    return {
+      validate: async (event) => {
+        const entity = event.entity.toUpperCase();
+        const operation = event.operation.toLowerCase();
+        logger.debug(`🩺 ${entity}.${operation}.${event.payload?._id || '*'}`);
+        try {
+          const fn = getOperation(entity, operation);
+          if (fn.payloadSchema) {
+            fn.payloadSchema.parse(event.payload);
+          }
+        } catch (e) {
+          throw new Error(`🤧 ${entity}.${operation}.${event.payload?._id || '*'} (${e.message})`);
+        }
+      },
+      prepare: async (event, unchainedAPI: { modules: Modules; services: Services }) => {
+        const entity = event.entity.toUpperCase();
+        const operation = event.operation.toLowerCase();
+
+        const handler = getOperation(entity, operation);
+
+        const payloadId = event.payload?._id || 'global';
+
+        logger.debug(`🏃‍♂️ ${entity}.${operation}.${payloadId}`);
+
+        try {
+          await handler(event.payload, { bulk, logger, ...options }, unchainedAPI);
+          if (!processedOperations[entity]) processedOperations[entity] = {};
+          if (!processedOperations[entity][operation]) processedOperations[entity][operation] = [];
+          processedOperations[entity][operation].push(payloadId);
+          logger.debug(`👍 ${entity}.${operation}.${payloadId}`);
+        } catch (e) {
+          logger.debug(`💥 ${entity}.${operation}.${payloadId} (${e.message})`);
+          preparationIssues.push({
+            operation,
+            entity,
+            payloadId,
+            errorCode: e.name,
+            errorMessage: e.message,
+          });
+        } finally {
+          logger.debug(`🏁 ${entity}.${operation}.${payloadId}`);
+        }
+      },
+      execute: async () => {
+        logger.debug(`🍔 ${Object.keys(bulkOperations).join(', ')}`);
+        const processedBulkOperations = await Promise.allSettled(
+          Object.values(bulkOperations).map(async (o: any) => o.execute()),
+        );
+        const operationResults = {
+          processedOperations,
+          processedBulkOperations,
+        };
+        if (preparationIssues?.length) {
+          logger.warn(
+            `${preparationIssues.length} issues occured while preparing events, import finished with errors`,
+          );
+          const errors = { preparationIssues };
+          return [operationResults, errors];
+        }
+        return [operationResults, null];
+      },
+      invalidateCaches: async (unchainedAPI: { modules: Modules; services: Services }) => {
+        if (skipCacheInvalidation) return;
+        await unchainedAPI.modules.assortments.invalidateCache({}, { skipUpstreamTraversal: true });
+        await unchainedAPI.services.filters.invalidateFilterCache();
+      },
+    };
+  };
+
+  const validateEventStream = async (readStream: Readable) => {
+    const bulkImporter = createBulkImporter({});
+    await pipeline(
+      readStream,
+      JSONStream.parse('events.*'),
+      async function* (source: AsyncIterable<any>, { signal }) {
+        for await (const event of source) {
+          if (signal.aborted) break;
+          await bulkImporter.validate(event);
+        }
+        yield true;
+      },
+    );
+  };
+
+  const pipeEventStream = async (
+    readStream: Readable,
+    bulkImporter: ReturnType<typeof createBulkImporter>,
+    unchainedAPI: { modules: Modules; services: Services },
+  ) => {
+    await pipeline(
+      readStream,
+      new PassThrough({
+        highWaterMark: 100,
+      }),
+      JSONStream.parse('events.*'),
+      async function* (source) {
+        for await (const event of source) {
+          await bulkImporter.prepare(event, unchainedAPI);
+        }
+        yield true;
+      },
+    );
+  };
+
+  return {
+    createBulkImporter,
+    validateEventStream,
+    pipeEventStream,
+  };
+}
+
+export type BulkImporter = ReturnType<typeof createBulkImporterFactory>;
+
+export { AssortmentHandlers, ProductHandlers, FilterHandlers };

package/src/bulk-importer/handlers/assortment/create.ts

@@ -0,0 +1,118 @@
+import convertTagsToLowerCase from '../utils/convertTagsToLowerCase.js';
+import upsertAssortmentChildren, { AssortmentLinkSchema } from './upsertAssortmentChildren.js';
+import upsertAssortmentFilters, { AssortmentFilterSchema } from './upsertAssortmentFilters.js';
+import upsertAssortmentProducts, { AssortmentProductSchema } from './upsertAssortmentProducts.js';
+import upsertMedia, { MediaSchema } from './upsertMedia.js';
+import { Modules } from '../../../modules.js';
+import { Services } from '../../../services/index.js';
+import { z } from 'zod';
+
+export const AssortmentCreatePayloadSchema = z.object({
+  _id: z.string(),
+  specification: z.object({
+    isActive: z.boolean(),
+    isBase: z.boolean().optional(),
+    isRoot: z.boolean().optional(),
+    sequence: z.number(),
+    tags: z.array(z.string()).optional(),
+    meta: z.record(z.unknown()).optional(),
+    content: z
+      .record(
+        z.string(), // locale
+        z.object({
+          title: z.string().optional(),
+          subtitle: z.string().optional(),
+          slug: z.string().optional(),
+        }),
+      )
+      .optional(),
+  }),
+  media: z.array(MediaSchema).optional(),
+  products: z.array(AssortmentProductSchema).optional(),
+  children: z.array(AssortmentLinkSchema).optional(),
+  filters: z.array(AssortmentFilterSchema).optional(),
+});
+
+export default async function createAssortment(
+  payload: z.infer<typeof AssortmentCreatePayloadSchema>,
+  { logger, createShouldUpsertIfIDExists },
+  unchainedAPI: { modules: Modules; services: Services },
+) {
+  const { modules } = unchainedAPI;
+  const { media, specification, products, children, filters, _id } = payload;
+  if (!specification) throw new Error(`Specification is required when creating new assortment ${_id}`);
+
+  if (!specification.content)
+    throw new Error(`Assortment content is required when creating new assortment${_id}`);
+
+  if (specification.tags) {
+    specification.tags = convertTagsToLowerCase(specification.tags)!;
+  }
+
+  logger.debug('create assortment object', specification);
+  try {
+    await modules.assortments.create({ ...specification, _id } as any);
+  } catch (e) {
+    if (!createShouldUpsertIfIDExists) throw e;
+    logger.debug('entity already exists, falling back to update', specification);
+    await modules.assortments.update(_id, {
+      ...specification,
+    });
+  }
+
+  if (!(await modules.assortments.assortmentExists({ assortmentId: _id }))) {
+    throw new Error(`Can't create assortment ${_id}, fields missing?`);
+  }
+
+  if (specification.content) {
+    logger.debug('create localized content for assortment', specification.content);
+    await modules.assortments.texts.updateTexts(
+      _id,
+      Object.entries(specification.content).map(([locale, localizedData]: [string, any]) => {
+        return {
+          locale,
+          ...localizedData,
+        };
+      }),
+    );
+  }
+
+  logger.debug('create assortment products', products);
+  await upsertAssortmentProducts(
+    {
+      products: products || [],
+      assortmentId: _id,
+    },
+    unchainedAPI,
+  );
+
+  logger.debug('create assortment children', children);
+  await upsertAssortmentChildren(
+    {
+      children: children || [],
+      assortmentId: _id,
+    },
+    unchainedAPI,
+  );
+
+  logger.debug('create assortment filters', filters);
+  await upsertAssortmentFilters(
+    {
+      filters: filters || [],
+      assortmentId: _id,
+    },
+    unchainedAPI,
+  );
+
+  logger.debug('create assortment media', media);
+  await upsertMedia({ media: media || [], assortmentId: _id }, unchainedAPI);
+
+  return {
+    entity: 'ASSORTMENT',
+    operation: 'create',
+    _id,
+    success: true,
+  };
+}
+
+createAssortment.payloadSchema = AssortmentCreatePayloadSchema;

package/src/bulk-importer/handlers/assortment/index.ts

@@ -0,0 +1,3 @@
+export { default as create } from './create.js';
+export { default as update } from './update.js';
+export { default as remove } from './remove.js';

package/src/bulk-importer/handlers/assortment/remove.ts

@@ -0,0 +1,26 @@
+import { z } from 'zod';
+import { Modules } from '../../../modules.js';
+
+export const AssortmentRemovePayloadSchema = z.object({
+  _id: z.string(),
+});
+
+export default async function removeAssortment(
+  payload: z.infer<typeof AssortmentRemovePayloadSchema>,
+  { logger },
+  unchainedAPI: { modules: Modules },
+) {
+  const { modules } = unchainedAPI;
+  const { _id } = payload;
+  logger.debug('remove assortment');
+  const deletedAssortment = await modules.assortments.delete(_id, { skipInvalidation: true });
+
+  return {
+    entity: 'ASSORTMENT',
+    operation: 'remove',
+    _id,
+    success: Boolean(deletedAssortment),
+  };
+}
+
+removeAssortment.payloadSchema = AssortmentRemovePayloadSchema;

package/src/bulk-importer/handlers/assortment/update.ts

@@ -0,0 +1,133 @@
+import { z } from 'zod';
+import upsertAssortmentProducts, { AssortmentProductSchema } from './upsertAssortmentProducts.js';
+import upsertAssortmentChildren, { AssortmentLinkSchema } from './upsertAssortmentChildren.js';
+import upsertAssortmentFilters, { AssortmentFilterSchema } from './upsertAssortmentFilters.js';
+import upsertMedia, { MediaSchema } from './upsertMedia.js';
+import convertTagsToLowerCase from '../utils/convertTagsToLowerCase.js';
+import createAssortment, { AssortmentCreatePayloadSchema } from './create.js';
+import { Modules } from '../../../modules.js';
+import { Services } from '../../../services/index.js';
+
+export const AssortmentUpdatePayloadSchema = z.object({
+  _id: z.string(),
+  specification: z
+    .object({
+      isActive: z.boolean().optional(),
+      isBase: z.boolean().optional(),
+      isRoot: z.boolean().optional(),
+      sequence: z.number().optional(),
+      tags: z.array(z.string()).optional(),
+      meta: z.record(z.unknown()).optional(),
+      content: z
+        .record(
+          z.string(), // locale
+          z.object({
+            title: z.string().optional(),
+            subtitle: z.string().optional(),
+            slug: z.string().optional(),
+          }),
+        )
+        .optional(),
+    })
+    .optional(),
+  media: z.array(MediaSchema).optional(),
+  products: z.array(AssortmentProductSchema).optional(),
+  children: z.array(AssortmentLinkSchema).optional(),
+  filters: z.array(AssortmentFilterSchema).optional(),
+});
+
+export default async function updateAssortment(
+  payload: z.infer<typeof AssortmentUpdatePayloadSchema>,
+  { logger, createShouldUpsertIfIDExists, updateShouldUpsertIfIDNotExists },
+  unchainedAPI: { modules: Modules; services: Services },
+) {
+  const { modules } = unchainedAPI;
+  const { media, specification, products, children, filters, _id } = payload;
+
+  if (!(await modules.assortments.assortmentExists({ assortmentId: _id }))) {
+    if (updateShouldUpsertIfIDNotExists && specification) {
+      return createAssortment(
+        {
+          _id,
+          specification,
+          media,
+          products,
+          children,
+          filters,
+        } as z.infer<typeof AssortmentCreatePayloadSchema>,
+        { logger, createShouldUpsertIfIDExists },
+        unchainedAPI,
+      );
+    }
+    throw new Error(`Can't update non-existing assortment ${_id}`);
+  }
+
+  if (specification) {
+    logger.debug('update assortment object', specification);
+
+    if (specification.tags) {
+      specification.tags = convertTagsToLowerCase(specification.tags!)!;
+    }
+
+    await unchainedAPI.modules.assortments.update(_id, { ...specification });
+
+    if (specification.content) {
+      logger.debug('replace localized content for assortment', specification.content);
+      await modules.assortments.texts.updateTexts(
+        _id,
+        Object.entries(specification.content).map(([locale, localizedData]: [string, any]) => {
+          return {
+            locale,
+            ...localizedData,
+          };
+        }),
+      );
+    }
+  }
+
+  if (products) {
+    logger.debug('update product products', products);
+    await upsertAssortmentProducts(
+      {
+        products: products || [],
+        assortmentId: _id,
+      },
+      unchainedAPI,
+    );
+  }
+
+  if (children) {
+    logger.debug('update assortment children', children);
+    await upsertAssortmentChildren(
+      {
+        children: children || [],
+        assortmentId: _id,
+      },
+      unchainedAPI,
+    );
+  }
+
+  if (filters) {
+    logger.debug('update assortment filters', filters);
+    await upsertAssortmentFilters(
+      {
+        filters: filters || [],
+        assortmentId: _id,
+      },
+      unchainedAPI,
+    );
+  }
+  if (media) {
+    logger.debug('update assortment media', media);
+    await upsertMedia({ media: media || [], assortmentId: _id }, unchainedAPI);
+  }
+
+  return {
+    entity: 'ASSORTMENT',
+    operation: 'update',
+    _id,
+    success: true,
+  };
+}
+
+updateAssortment.payloadSchema = AssortmentUpdatePayloadSchema;

package/src/bulk-importer/handlers/assortment/upsertAssortmentChildren.ts

@@ -0,0 +1,77 @@
+import { z } from 'zod';
+import convertTagsToLowerCase from '../utils/convertTagsToLowerCase.js';
+import { Modules } from '../../../modules.js';
+
+export const AssortmentLinkSchema = z.object({
+  _id: z.string().optional(),
+  assortmentId: z.string(),
+  tags: z.array(z.string()).optional(),
+  sortKey: z.number().optional(),
+  meta: z.record(z.unknown()).optional(),
+});
+
+const upsert = async (
+  assortmentLink: {
+    _id?: string;
+    parentAssortmentId: string;
+    childAssortmentId: string;
+    tags: string[];
+    sortKey?: number;
+  },
+  { modules }: { modules: Modules },
+) => {
+  if (
+    !(await modules.assortments.assortmentExists({
+      assortmentId: assortmentLink.childAssortmentId,
+    }))
+  ) {
+    throw new Error(`Can't link non-existing assortment ${assortmentLink.childAssortmentId}`);
+  }
+  try {
+    const newAssortmentLink = await modules.assortments.links.create(assortmentLink, {
+      skipInvalidation: true,
+    });
+    return newAssortmentLink!;
+  } catch {
+    return (await modules.assortments.links.update(assortmentLink._id!, assortmentLink, {
+      skipInvalidation: true,
+    }))!;
+  }
+};
+
+export default async (
+  {
+    children,
+    assortmentId: parentAssortmentId,
+  }: {
+    children: z.infer<typeof AssortmentLinkSchema>[];
+    assortmentId: string;
+  },
+  unchainedAPI: { modules: Modules },
+) => {
+  const { modules } = unchainedAPI;
+  const assortmentLinkIds = await Promise.all(
+    children.map(
+      async ({ assortmentId: childAssortmentId, sortKey, tags: tagsMixedCase }, sortKeyForced) => {
+        const tags = tagsMixedCase ? convertTagsToLowerCase(tagsMixedCase)! : [];
+        const assortmentLink = await upsert(
+          {
+            sortKey: sortKey ?? sortKeyForced,
+            tags,
+            parentAssortmentId,
+            childAssortmentId,
+          },
+          unchainedAPI,
+        );
+        return assortmentLink._id;
+      },
+    ),
+  );
+  await modules.assortments.links.deleteMany(
+    {
+      _id: { $nin: assortmentLinkIds },
+      parentAssortmentId,
+    },
+    { skipInvalidation: true },
+  );
+};

package/src/bulk-importer/handlers/assortment/upsertAssortmentFilters.ts

@@ -0,0 +1,70 @@
+import { z } from 'zod';
+import convertTagsToLowerCase from '../utils/convertTagsToLowerCase.js';
+import { Modules } from '../../../modules.js';
+
+export const AssortmentFilterSchema = z.object({
+  _id: z.string().optional(),
+  filterId: z.string(),
+  tags: z.array(z.string()).optional(),
+  sortKey: z.number().optional(),
+  meta: z.record(z.unknown()).optional(),
+});
+
+const upsert = async (
+  assortmentFilter: {
+    _id?: string;
+    filterId: string;
+    tags: string[];
+    assortmentId: string;
+    sortKey: number;
+  },
+  { modules }: { modules: Modules },
+) => {
+  if (
+    !(await modules.filters.filterExists({
+      filterId: assortmentFilter.filterId,
+    }))
+  ) {
+    throw new Error(`Can't link non-existing filter ${assortmentFilter.filterId}`);
+  }
+  try {
+    const newAssortmentFilter = await modules.assortments.filters.create(assortmentFilter);
+    return newAssortmentFilter;
+  } catch {
+    return await modules.assortments.filters.update(assortmentFilter._id!, assortmentFilter);
+  }
+};
+
+export default async (
+  {
+    filters,
+    assortmentId,
+  }: {
+    filters: z.infer<typeof AssortmentFilterSchema>[];
+    assortmentId: string;
+  },
+  unchainedAPI: { modules: Modules },
+) => {
+  const { modules } = unchainedAPI;
+  const assortmentFilterIds = await Promise.all(
+    filters.map(async ({ tags: tagsMixedCase, sortKey, ...rest }, forcedSortKey) => {
+      const tags = tagsMixedCase ? convertTagsToLowerCase(tagsMixedCase)! : [];
+
+      const assortmentFilter = await upsert(
+        {
+          ...rest,
+          assortmentId,
+          tags,
+          sortKey: sortKey ?? forcedSortKey,
+        },
+        unchainedAPI,
+      );
+      return assortmentFilter!._id;
+    }),
+  );
+
+  await modules.assortments.filters.deleteMany({
+    _id: { $nin: assortmentFilterIds },
+    assortmentId,
+  });
+};