@erxes/api-shared 0.0.0-stage → 3.2.19-2

package/src/core-modules/import-export/utils/importExportError.ts

@@ -0,0 +1,71 @@
+export type ImportExportStage =
+  | 'LOAD_DOCUMENT'
+  | 'FETCH_FILE'
+  | 'FETCH_HEADERS'
+  | 'PARSE_ROWS'
+  | 'PROCESS_BATCH'
+  | 'WRITE_TEMP_FILE'
+  | 'FINALIZE_UPLOAD'
+  | 'SAVE_RESULT'
+  | 'UPDATE_STATUS';
+
+export class ImportExportError extends Error {
+  stage: ImportExportStage;
+  code: string;
+  retryable: boolean;
+  cause?: unknown;
+
+  constructor({
+    stage,
+    message,
+    code = 'IMPORT_EXPORT_STAGE_FAILED',
+    retryable = true,
+    cause,
+  }: {
+    stage: ImportExportStage;
+    message: string;
+    code?: string;
+    retryable?: boolean;
+    cause?: unknown;
+  }) {
+    super(message);
+    this.name = 'ImportExportError';
+    this.stage = stage;
+    this.code = code;
+    this.retryable = retryable;
+    this.cause = cause;
+  }
+}
+
+export const withImportExportStage = async <T>({
+  stage,
+  run,
+  fallbackMessage,
+  code = 'IMPORT_EXPORT_STAGE_FAILED',
+  retryable = true,
+}: {
+  stage: ImportExportStage;
+  run: () => Promise<T>;
+  fallbackMessage: string;
+  code?: string;
+  retryable?: boolean;
+}): Promise<T> => {
+  try {
+    return await run();
+  } catch (error) {
+    if (error instanceof ImportExportError) {
+      throw error;
+    }
+
+    const message =
+      error instanceof Error && error.message ? error.message : fallbackMessage;
+
+    throw new ImportExportError({
+      stage,
+      message,
+      code,
+      retryable,
+      cause: error,
+    });
+  }
+};

package/src/core-modules/import-export/utils/importExportRuntime.ts

@@ -0,0 +1,96 @@
+import { ImportExportError } from './importExportError';
+
+export const getImportExportJobOptions = ({
+  attempts = 2,
+  backoffDelay = 2000,
+}: {
+  attempts?: number;
+  backoffDelay?: number;
+}) => {
+  return {
+    attempts,
+    backoff: {
+      type: 'exponential' as const,
+      delay: backoffDelay,
+    },
+    removeOnComplete: 50,
+    removeOnFail: 100,
+  };
+};
+
+export const toTerminalImportExportError = (error: unknown) => {
+  if (error instanceof ImportExportError) {
+    return {
+      code: error.code,
+      stage: error.stage,
+      retryable: error.retryable,
+      message: error.message,
+    };
+  }
+
+  return {
+    code: 'IMPORT_EXPORT_UNKNOWN',
+    stage: 'UPDATE_STATUS',
+    retryable: true,
+    message:
+      error instanceof Error ? error.message : 'Unknown import/export error',
+  };
+};
+
+export const logImportExportEvent = ({
+  level = 'info',
+  entity,
+  id,
+  subdomain,
+  stage,
+  event,
+  extra,
+}: {
+  level?: 'info' | 'warn' | 'error';
+  entity: 'import' | 'export';
+  id: string;
+  subdomain: string;
+  stage: string;
+  event: string;
+  extra?: Record<string, unknown>;
+}) => {
+  const line = JSON.stringify({
+    scope: 'import-export',
+    entity,
+    id,
+    subdomain,
+    stage,
+    event,
+    ...(extra || {}),
+  });
+
+  if (level === 'error') {
+    console.error(line);
+    return;
+  }
+
+  if (level === 'warn') {
+    console.warn(line);
+    return;
+  }
+
+  console.log(line);
+};
+
+export const safeCleanup = async ({
+  label,
+  run,
+}: {
+  label: string;
+  run: () => Promise<void>;
+}) => {
+  try {
+    await run();
+  } catch (error: any) {
+    console.warn(
+      `[ImportExport] Cleanup failed at ${label}: ${
+        error?.message || 'Unknown error'
+      }`,
+    );
+  }
+};

package/src/core-modules/import-export/utils/importUtils.ts

@@ -0,0 +1,59 @@
+import { Readable } from 'stream';
+import { parse as csvParse } from 'csv-parse';
+
+/**
+ * Stream a CSV file row by row.
+ *
+ * Memory usage is O(1 row) instead of O(file size): bytes flow from the
+ * source stream (e.g. an S3 GetObject stream) through csv-parse, and each
+ * row is yielded as soon as it is parsed.
+ *
+ * Error propagation: if the source stream errors mid-download (e.g. S3
+ * connection drop) we destroy the csv-parse transform with the same error,
+ * which causes the outer `for await` in the worker to throw. The import
+ * job then correctly transitions to `failed` instead of silently finishing
+ * with partial data or crashing the worker process.
+ */
+export async function* processCSVStream(
+  input: Readable,
+): AsyncGenerator<string[], void, unknown> {
+  const parser = csvParse({
+    bom: true, // strip UTF-8 BOM automatically
+    relax_quotes: true, // tolerate loosely quoted fields
+    skip_empty_lines: true,
+    trim: true,
+  });
+
+  input.on('error', (err) => parser.destroy(err));
+  input.pipe(parser);
+
+  try {
+    for await (const row of parser as AsyncIterable<string[]>) {
+      yield row;
+    }
+  } finally {
+    // If the consumer breaks out of the loop early, make sure both streams
+    // are torn down so we don't leak sockets / file descriptors.
+    if (!parser.destroyed) parser.destroy();
+    if (!input.destroyed) input.destroy();
+  }
+}
+
+/**
+ * Count the data rows of a CSV file (header excluded).
+ *
+ * The worker needs a real denominator before it starts: without it,
+ * progress is computed against "rows read so far", which makes the bar
+ * jump backwards on every batch and pins the ETA near zero. Counting is a
+ * separate streaming pass — it never holds more than one row in memory.
+ */
+export async function countCsvDataRows(input: Readable): Promise<number> {
+  const rowIterator = processCSVStream(input);
+  let rows = 0;
+
+  while (!(await rowIterator.next()).done) {
+    rows++;
+  }
+
+  return Math.max(rows - 1, 0);
+}

package/src/core-modules/import-export/utils/progressUpdate.ts

@@ -0,0 +1,25 @@
+type SafeProgressUpdateArgs = {
+  entity: 'import' | 'export';
+  id: string;
+  subdomain: string;
+  stage: string;
+  update: () => Promise<any>;
+};
+
+export const safeProgressUpdate = async ({
+  entity,
+  id,
+  subdomain,
+  stage,
+  update,
+}: SafeProgressUpdateArgs): Promise<void> => {
+  try {
+    await update();
+  } catch (error: any) {
+    console.error(
+      `[ImportExport] Failed to update ${entity} progress at stage "${stage}" for ${subdomain}/${id}: ${
+        error?.message || 'Unknown error'
+      }`,
+    );
+  }
+};

package/src/core-modules/import-export/utils/tempWorkspace.ts

@@ -0,0 +1,34 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+type TempWorkspaceKind = 'import' | 'export';
+
+export interface ImportExportTempWorkspace {
+  dirPath: string;
+  createFilePath(fileName: string): string;
+  cleanup(): Promise<void>;
+}
+
+export const createImportExportTempWorkspace = async ({
+  kind,
+}: {
+  kind: TempWorkspaceKind;
+}): Promise<ImportExportTempWorkspace> => {
+  const dirPath = await fs.promises.mkdtemp(
+    path.join(os.tmpdir(), `erxes-${kind}-`),
+  );
+
+  return {
+    dirPath,
+    createFilePath(fileName: string) {
+      return path.join(dirPath, fileName);
+    },
+    async cleanup() {
+      await fs.promises.rm(dirPath, {
+        recursive: true,
+        force: true,
+      });
+    },
+  };
+};

package/src/core-modules/import-export/worker.ts

@@ -0,0 +1,215 @@
+import { AnyProcedure, initTRPC } from '@trpc/server';
+import * as trpcExpress from '@trpc/server/adapters/express';
+import { Express } from 'express';
+import { nanoid } from 'nanoid';
+import { z } from 'zod';
+import {
+  createMQWorkerWithListeners,
+  createTRPCContext,
+  getEnv,
+  initializePluginConfig,
+  redis,
+} from '../../utils';
+import { TExportHandlers, ImportExportConfigs, TImportHandlers } from './types';
+import { createCoreImportClient } from './utils/createCoreImportClient';
+import { createImportBatchProcessor } from './utils/importBatchProcess';
+import { createExportBatchProcessor } from './utils/exportBatchProcess';
+
+const IMPORT_QUEUE = 'import-processor';
+const EXPORT_QUEUE = 'export-processor';
+
+const startedWorkers = new Set<string>();
+
+const parsePositiveInteger = (value: string, fallback: number): number => {
+  const parsed = Number.parseInt(value, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+};
+
+const getImportExportWorkerOptions = (kind: 'import' | 'export') => {
+  const upperKind = kind.toUpperCase();
+  const concurrency = parsePositiveInteger(
+    getEnv({
+      name: `IMPORT_EXPORT_${upperKind}_CONCURRENCY`,
+      defaultValue: '1',
+    }),
+    1,
+  );
+
+  const limiterMax = parsePositiveInteger(
+    getEnv({
+      name: `IMPORT_EXPORT_${upperKind}_LIMITER_MAX`,
+      defaultValue: '0',
+    }),
+    0,
+  );
+
+  const limiterDuration = parsePositiveInteger(
+    getEnv({
+      name: `IMPORT_EXPORT_${upperKind}_LIMITER_DURATION_MS`,
+      defaultValue: '0',
+    }),
+    0,
+  );
+
+  return {
+    concurrency,
+    ...(limiterMax > 0 && limiterDuration > 0
+      ? {
+          limiter: {
+            max: limiterMax,
+            duration: limiterDuration,
+          },
+        }
+      : {}),
+  };
+};
+
+const generateImportExportRouter = (
+  {
+    getImportHeaders,
+    batchSkipRow,
+  }: TImportHandlers | undefined = {} as TImportHandlers,
+  { getExportHeaders }: TExportHandlers | undefined = {} as TExportHandlers,
+) => {
+  const routerConfig: Partial<
+    Record<
+      'getImportHeaders' | 'getExportHeaders' | 'batchSkipRow',
+      AnyProcedure
+    >
+  > = {};
+  const trpcRouter = initTRPC
+    .context<{ subdomain: string; processId: string }>()
+    .create();
+  if (getImportHeaders) {
+    routerConfig.getImportHeaders = trpcRouter.procedure
+      .input(
+        z.object({
+          subdomain: z.string(),
+          data: z.object({
+            moduleName: z.string(),
+            collectionName: z.string(),
+          }),
+        }),
+      )
+      .query(async ({ ctx, input }) => {
+        return await getImportHeaders(input, ctx);
+      });
+  }
+
+  if (getExportHeaders) {
+    routerConfig.getExportHeaders = trpcRouter.procedure
+      .input(
+        z.object({
+          subdomain: z.string(),
+          data: z.object({
+            moduleName: z.string(),
+            collectionName: z.string(),
+            filters: z.record(z.any()).optional(),
+          }),
+        }),
+      )
+      .query(async ({ ctx, input }) => {
+        return await getExportHeaders(input, ctx);
+      });
+  }
+
+  if (batchSkipRow) {
+    routerConfig.batchSkipRow = trpcRouter.procedure
+      .input(
+        z.object({
+          subdomain: z.string(),
+          data: z.object({
+            moduleName: z.string(),
+            collectionName: z.string(),
+            rowData: z.object({}),
+          }),
+        }),
+      )
+      .query(async ({ ctx, input }) => {
+        return await batchSkipRow(input, ctx);
+      });
+  }
+
+  const trpcMiddleware = trpcExpress.createExpressMiddleware({
+    router: trpcRouter.router(routerConfig),
+    createContext: createTRPCContext(async (_subdomain, context) => {
+      const processId = nanoid(12);
+      context.processId = processId;
+      return context;
+    }),
+  });
+
+  return trpcMiddleware;
+};
+
+export const startImportExportWorker = ({
+  pluginName,
+  config,
+  app,
+}: {
+  pluginName: string;
+  config: ImportExportConfigs;
+  app: Express;
+}) => {
+  if (startedWorkers.has(pluginName)) {
+    return;
+  }
+
+  const { import: importConfig, export: exportConfig } = config || {};
+
+  initializePluginConfig(pluginName, 'importExport', {
+    import: {
+      configured: !!importConfig,
+      hasGetImportHeaders: !!importConfig?.getImportHeaders,
+      hasInsertImportRows: !!importConfig?.insertImportRows,
+      types: importConfig?.types || [],
+      hasSkipBatch: !!importConfig?.batchSkipRow,
+    },
+    export: {
+      configured: !!exportConfig,
+      hasGetExportHeaders: !!exportConfig?.getExportHeaders,
+      hasGetExportData: !!exportConfig?.getExportData,
+      types: exportConfig?.types || [],
+    },
+  });
+
+  const coreClient = createCoreImportClient();
+  const trpcMiddleware = generateImportExportRouter(importConfig, exportConfig);
+
+  if (exportConfig) {
+    const exportWorkerOptions = getImportExportWorkerOptions('export');
+    createMQWorkerWithListeners(
+      pluginName,
+      EXPORT_QUEUE,
+      createExportBatchProcessor(exportConfig, coreClient, pluginName),
+      redis,
+      () => {
+        console.log(
+          `[ImportExport] Export worker ready for queue ${pluginName}-${EXPORT_QUEUE}`,
+        );
+        exportConfig.whenReady?.();
+      },
+      exportWorkerOptions,
+    );
+  }
+
+  if (importConfig) {
+    const importWorkerOptions = getImportExportWorkerOptions('import');
+    createMQWorkerWithListeners(
+      pluginName,
+      IMPORT_QUEUE,
+      createImportBatchProcessor(importConfig, coreClient, pluginName),
+      redis,
+      () => {
+        console.log(
+          `[ImportExport] Worker ready for queue ${pluginName}-${IMPORT_QUEUE}`,
+        );
+        importConfig.whenReady?.();
+      },
+      importWorkerOptions,
+    );
+  }
+
+  startedWorkers.add(pluginName);
+  app.use('/importExport', trpcMiddleware);
+};

package/src/core-modules/import-export/zodSchemas.ts

@@ -0,0 +1,63 @@
+import { z } from 'zod';
+import { TImportExportProducers } from './types';
+
+export const InsertImportRowsInput = z.object({
+  moduleName: z.string(),
+  collectionName: z.string(),
+  rows: z.array(z.any()),
+  userId: z.string(),
+});
+export const GetExportHeadersInput = z.object({
+  moduleName: z.string(),
+  collectionName: z.string(),
+  filters: z.record(z.unknown()).optional(),
+});
+export const BatchSkipRowInput = z.object({
+  moduleName: z.string(),
+  collectionName: z.string(),
+  rowData: z.record(z.any()),
+});
+
+export type TGetExportHeadersInput = z.infer<typeof GetExportHeadersInput>;
+
+export const GetExportDataInput = z.object({
+  moduleName: z.string(),
+  collectionName: z.string(),
+  cursor: z.string().optional(),
+  limit: z.number(),
+  filters: z.record(z.unknown()).optional(),
+  ids: z.array(z.string()).optional(),
+});
+
+export type TGetExportDataInput = z.infer<typeof GetExportDataInput>;
+type TInsertImportRowsInput = z.infer<typeof InsertImportRowsInput>;
+export type TBatchSkipRowProducerInput = z.infer<typeof BatchSkipRowInput>;
+
+export const InsertImportRowsOutput = z.object({
+  successRows: z.array(z.any()),
+  errorRows: z.array(z.any()),
+});
+
+export const GetImportHeadersInput = z.object({
+  moduleName: z.string(),
+  collectionName: z.string(),
+});
+
+export type TGetImportHeadersInput = z.infer<typeof GetImportHeadersInput>;
+
+export const GetImportHeadersOutput = z.array(
+  z.object({
+    label: z.string(),
+    key: z.string(),
+  }),
+);
+
+export type TInsertImportRowsOutput = z.infer<typeof InsertImportRowsOutput>;
+
+export type TImportExportProducersInput = {
+  [TImportExportProducers.INSERT_IMPORT_ROWS]: TInsertImportRowsInput;
+  [TImportExportProducers.GET_IMPORT_HEADERS]: TGetImportHeadersInput;
+  [TImportExportProducers.BATCH_SKIP_ROW]: TBatchSkipRowProducerInput;
+  [TImportExportProducers.GET_EXPORT_HEADERS]: TGetExportHeadersInput;
+  [TImportExportProducers.GET_EXPORT_DATA]: TGetExportDataInput;
+};

package/src/core-modules/index.ts

@@ -0,0 +1,15 @@
+export * from './permissions';
+export * from './users';
+export * from './common';
+export * from './apps';
+export * from './automations';
+export * from './segments';
+export * from './forms';
+export * from './properties';
+export * from './logs';
+export * from './notifications';
+export * from './approval';
+export * from './email-content';
+export * from './client-portals';
+export * from './import-export';
+export * from './templates';