@erxes/api-shared 0.0.0-stage → 3.2.19-1

package/src/core-modules/import-export/utils/importBatchProcess.ts

@@ -0,0 +1,780 @@
+import { Job, UnrecoverableError } from 'bullmq';
+import { nanoid } from 'nanoid';
+import * as fs from 'fs';
+import { splitType } from '../../../core-modules/automations';
+import { readFileStreamFromStorage } from '../../../utils/file/read';
+import { IImportExportContext, TImportHandlers, ImportJobData } from '../types';
+import { CoreImportClient } from './createCoreImportClient';
+import { ImportExportError, withImportExportStage } from './importExportError';
+import {
+  logImportExportEvent,
+  safeCleanup,
+  toTerminalImportExportError,
+} from './importExportRuntime';
+import { matchImportHeaders } from './headerMatcher';
+import { countCsvDataRows, processCSVStream } from './importUtils';
+import { safeProgressUpdate } from './progressUpdate';
+import {
+  ImportExportTempWorkspace,
+  createImportExportTempWorkspace,
+} from './tempWorkspace';
+
+type ImportErrorRow = Record<string, any>;
+
+interface ImportErrorRowWriter {
+  writeRows(rows: ImportErrorRow[]): Promise<void>;
+  finalize(): Promise<void>;
+  cleanup(): Promise<void>;
+  getErrorRows(): ImportErrorRow[];
+  hasErrors(): boolean;
+}
+
+const createContext = async (
+  subdomain: string,
+  baseContext: IImportExportContext,
+  config: TImportHandlers,
+): Promise<IImportExportContext> => {
+  // if (config.createContext) {
+  //   return await config.createContext(subdomain, baseContext);
+  // }
+
+  return baseContext;
+};
+
+const escapeCsvField = (
+  field: string | number | boolean | null | undefined,
+): string => {
+  const value =
+    field === null || field === undefined
+      ? ''
+      : typeof field === 'string'
+      ? field
+      : String(field);
+
+  if (value.includes('"') || value.includes(',') || value.includes('\n')) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+
+  return value;
+};
+
+const createImportErrorRowWriter = async ({
+  importId,
+  getCsvHeaders,
+  keyToHeaderMap,
+  tempWorkspace,
+}: {
+  importId: string;
+  getCsvHeaders: () => string[];
+  keyToHeaderMap: Record<string, string>;
+  tempWorkspace: ImportExportTempWorkspace;
+}): Promise<ImportErrorRowWriter> => {
+  const errorFilePath = tempWorkspace.createFilePath(
+    `import-errors-${importId}-${nanoid()}.csv`,
+  );
+
+  const stream = fs.createWriteStream(errorFilePath, { encoding: 'utf8' });
+  let initialized = false;
+  let streamError: Error | null = null;
+  const bufferedErrorRows: ImportErrorRow[] = [];
+
+  stream.on('error', (err) => {
+    streamError = err;
+  });
+
+  const writeChunk = async (chunk: string) => {
+    if (streamError) {
+      throw streamError;
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      const cleanupListeners = () => {
+        stream.off('error', onError);
+        stream.off('drain', onDrain);
+      };
+
+      const onError = (err: Error) => {
+        cleanupListeners();
+        reject(err);
+      };
+
+      const onDrain = () => {
+        cleanupListeners();
+        resolve();
+      };
+
+      stream.once('error', onError);
+
+      const canContinue = stream.write(chunk, (err) => {
+        if (err) {
+          cleanupListeners();
+          reject(err);
+        }
+      });
+
+      if (canContinue) {
+        cleanupListeners();
+        resolve();
+        return;
+      }
+
+      stream.once('drain', onDrain);
+    });
+  };
+
+  const ensureHeader = async () => {
+    if (initialized) {
+      return;
+    }
+
+    const finalHeaders = [...getCsvHeaders(), 'Error'];
+    await writeChunk(finalHeaders.map(escapeCsvField).join(',') + '\n');
+    initialized = true;
+  };
+
+  return {
+    async writeRows(rows) {
+      if (!rows.length) {
+        return;
+      }
+
+      await ensureHeader();
+
+      const csvHeaders = getCsvHeaders();
+      const headerToKeyMap: Record<string, string> = {};
+
+      Object.entries(keyToHeaderMap).forEach(([key, header]) => {
+        headerToKeyMap[header] = key;
+      });
+
+      const chunk =
+        rows
+          .map((row) => {
+            const errorMessage =
+              row?.error ||
+              row?.errorMessage ||
+              row?.message ||
+              'Unknown error';
+
+            const normalizedRow: ImportErrorRow = { error: errorMessage };
+
+            const values = csvHeaders.map((header) => {
+              // rows are keyed by field key, so the file's header has to be
+              // translated back into one before the value can be read
+              const lookupKey = headerToKeyMap[header] || header;
+              const value = row?.[lookupKey];
+              normalizedRow[lookupKey] = value;
+              return escapeCsvField(value);
+            });
+
+            bufferedErrorRows.push(normalizedRow);
+
+            values.push(escapeCsvField(errorMessage));
+            return values.join(',');
+          })
+          .join('\n') + '\n';
+
+      await writeChunk(chunk);
+    },
+
+    async finalize() {
+      if (streamError) {
+        throw streamError;
+      }
+
+      await new Promise<void>((resolve, reject) => {
+        stream.end((err: any) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+
+          resolve();
+        });
+      });
+
+      if (streamError) {
+        throw streamError;
+      }
+    },
+
+    async cleanup() {
+      if (!stream.destroyed) {
+        stream.destroy();
+      }
+    },
+
+    getErrorRows() {
+      return bufferedErrorRows;
+    },
+
+    hasErrors() {
+      return bufferedErrorRows.length > 0;
+    },
+  };
+};
+
+export const createImportBatchProcessor = (
+  config: TImportHandlers,
+  coreClient: CoreImportClient,
+  pluginName: string,
+) => {
+  return async (job: Job<ImportJobData>) => {
+    const { subdomain, data } = job.data;
+    const { importId, entityType, fileKey, userId } = data;
+    const [jobPluginName, moduleName, collectionName] = splitType(entityType);
+
+    if (jobPluginName !== pluginName) {
+      return;
+    }
+
+    if (!config.insertImportRows) {
+      throw new Error('insertImportRows handler is not defined');
+    }
+
+    if (!config.getImportHeaders) {
+      throw new Error('getImportHeaders handler is not defined');
+    }
+
+    const processId = nanoid(12);
+    const context = await createContext(
+      subdomain,
+      { subdomain, processId },
+      config,
+    );
+    let dataRowIndex = 0;
+    let totalRows = 0;
+    let processedRows = 0;
+    let successRows = 0;
+    let errorRows = 0;
+    const startedAt = Date.now();
+
+    logImportExportEvent({
+      entity: 'import',
+      id: importId,
+      subdomain,
+      stage: 'START',
+      event: 'job_started',
+      extra: {
+        entityType,
+        fileKey,
+      },
+    });
+
+    try {
+      await safeProgressUpdate({
+        entity: 'import',
+        id: importId,
+        subdomain,
+        stage: 'validating',
+        update: async () => {
+          await coreClient.updateImportProgress(subdomain, importId, {
+            status: 'validating',
+          });
+        },
+      });
+
+      const importDoc = await withImportExportStage({
+        stage: 'LOAD_DOCUMENT',
+        fallbackMessage: 'Failed to load import document',
+        retryable: false,
+        run: async () => await coreClient.getImport(subdomain, importId),
+      });
+      if (!importDoc) {
+        throw new ImportExportError({
+          stage: 'LOAD_DOCUMENT',
+          message: 'Import not found',
+          code: 'IMPORT_NOT_FOUND',
+          retryable: false,
+        });
+      }
+
+      const fileName = importDoc.fileName || fileKey;
+
+      if (!fileName.toLowerCase().endsWith('.csv')) {
+        throw new ImportExportError({
+          stage: 'LOAD_DOCUMENT',
+          message: `Only .csv files can be imported. Received "${fileName}".`,
+          code: 'UNSUPPORTED_FILE_TYPE',
+          retryable: false,
+        });
+      }
+
+      const resumeFromRow = importDoc.lastProcessedRow || 0;
+      const confirmedMapping = importDoc.columnMapping || [];
+
+      // A resumed job continues the earlier tallies instead of restarting them
+      if (resumeFromRow > 0) {
+        processedRows = importDoc.processedRows || 0;
+        successRows = importDoc.successRows || 0;
+        errorRows = importDoc.errorRows || 0;
+      }
+
+      totalRows = await withImportExportStage({
+        stage: 'FETCH_FILE',
+        fallbackMessage: 'Failed to read import file',
+        run: async () =>
+          await countCsvDataRows(
+            await readFileStreamFromStorage({ subdomain, key: fileKey }),
+          ),
+      });
+
+      await safeProgressUpdate({
+        entity: 'import',
+        id: importId,
+        subdomain,
+        stage: 'row-count',
+        update: async () => {
+          await coreClient.updateImportProgress(subdomain, importId, {
+            totalRows,
+          });
+        },
+      });
+
+      // Native Readable stream from storage — the file is never fully
+      // buffered in memory. Storage errors (NoSuchKey, connection drop,
+      // etc.) surface via the stream's 'error' event, which is forwarded
+      // into the parser by processCSVStream and then
+      // thrown out of the for-await loop below, landing in the outer catch.
+      const fileStream = await withImportExportStage({
+        stage: 'FETCH_FILE',
+        fallbackMessage: 'Failed to fetch import file',
+        run: async () =>
+          await readFileStreamFromStorage({
+            subdomain,
+            key: fileKey,
+          }),
+      });
+
+      const batchSize = 5000;
+      let batch: any[] = [];
+
+      await safeProgressUpdate({
+        entity: 'import',
+        id: importId,
+        subdomain,
+        stage: 'processing',
+        update: async () => {
+          await coreClient.updateImportProgress(subdomain, importId, {
+            status: 'processing',
+          });
+        },
+      });
+
+      const importHeaders = await withImportExportStage({
+        stage: 'FETCH_HEADERS',
+        fallbackMessage: 'Failed to fetch import headers',
+        retryable: false,
+        run: async () =>
+          await config.getImportHeaders(
+            {
+              subdomain,
+              data: {
+                moduleName,
+                collectionName,
+              },
+            },
+            context,
+          ),
+      });
+
+      const rowIterator = processCSVStream(fileStream);
+
+      let rowIndex = 0;
+      let headerRow: string[] = [];
+      const columnToKeyMap: Record<number, string> = {};
+      const keyToHeaderMap: Record<string, string> = {};
+
+      const getCsvHeaders = (): string[] => {
+        return headerRow && headerRow.length > 0
+          ? headerRow
+          : Object.values(keyToHeaderMap);
+      };
+
+      const tempWorkspace = await createImportExportTempWorkspace({
+        kind: 'import',
+      });
+
+      try {
+        const errorRowWriter = await withImportExportStage({
+          stage: 'WRITE_TEMP_FILE',
+          fallbackMessage: 'Failed to initialize import error writer',
+          run: async () =>
+            await createImportErrorRowWriter({
+              importId,
+              getCsvHeaders,
+              keyToHeaderMap,
+              tempWorkspace,
+            }),
+        });
+
+        for await (const row of rowIterator) {
+          if (rowIndex === 0) {
+            headerRow = row;
+
+            const knownKeys = new Set(importHeaders.map((h) => h.key));
+
+            // A mapping confirmed by the user wins over anything the matcher
+            // would guess; the matcher only runs for imports started without
+            // one (a resumed job, or an API caller that sent no mapping).
+            if (confirmedMapping.length) {
+              for (const column of confirmedMapping) {
+                if (!knownKeys.has(column.key)) {
+                  continue;
+                }
+
+                columnToKeyMap[column.index] = column.key;
+                keyToHeaderMap[column.key] = String(
+                  headerRow[column.index] || column.header || '',
+                ).trim();
+              }
+            } else {
+              for (const match of matchImportHeaders(
+                headerRow,
+                importHeaders,
+              )) {
+                if (match.status !== 'matched' || !match.key) {
+                  continue;
+                }
+
+                columnToKeyMap[match.index] = match.key;
+                keyToHeaderMap[match.key] = match.header;
+              }
+            }
+
+            if (!Object.keys(columnToKeyMap).length) {
+              const expected = importHeaders
+                .slice(0, 10)
+                .map((h) => h.label)
+                .join(', ');
+
+              throw new ImportExportError({
+                stage: 'FETCH_HEADERS',
+                message:
+                  `None of the columns in the file matched a known field. ` +
+                  `File columns: ${headerRow.join(', ')}. ` +
+                  `Expected columns such as: ${expected}` +
+                  (importHeaders.length > 10 ? ', …' : ''),
+                code: 'NO_MATCHING_COLUMNS',
+                retryable: false,
+              });
+            }
+
+            rowIndex++;
+            continue;
+          }
+
+          dataRowIndex++;
+          rowIndex++;
+
+          if (dataRowIndex <= resumeFromRow) {
+            continue;
+          }
+
+          const rowData: Record<string, any> = {};
+          row.forEach((value: any, index: number) => {
+            if (
+              index >= 0 &&
+              value !== null &&
+              value !== undefined &&
+              value !== ''
+            ) {
+              const key = columnToKeyMap[index];
+              if (key) {
+                rowData[key] = value;
+              }
+            }
+          });
+
+          const shouldDeferBatchCut =
+            batch.length >= batchSize &&
+            config.batchSkipRow &&
+            (await config.batchSkipRow(
+              {
+                subdomain,
+                data: {
+                  moduleName,
+                  collectionName,
+                  rowData,
+                },
+              },
+              context,
+            ));
+
+          if (batch.length >= batchSize && !shouldDeferBatchCut) {
+            const result = await withImportExportStage({
+              stage: 'PROCESS_BATCH',
+              fallbackMessage: 'Failed to process import batch',
+              run: async () =>
+                await config.insertImportRows(
+                  {
+                    subdomain,
+                    data: {
+                      moduleName,
+                      collectionName,
+                      rows: batch,
+                      userId,
+                    },
+                  },
+                  context,
+                ),
+            });
+
+            const successIds =
+              result.successRows
+                ?.map((r: any) => r._id || r.id)
+                .filter(Boolean) || [];
+
+            successRows += successIds.length;
+            errorRows += result.errorRows.length;
+            processedRows += batch.length;
+
+            if (result.errorRows.length) {
+              await withImportExportStage({
+                stage: 'WRITE_TEMP_FILE',
+                fallbackMessage: 'Failed to write import error rows',
+                run: async () =>
+                  await errorRowWriter.writeRows(result.errorRows),
+              });
+            }
+
+            await safeProgressUpdate({
+              entity: 'import',
+              id: importId,
+              subdomain,
+              stage: 'batch-progress',
+              update: async () => {
+                await coreClient.updateImportProgress(subdomain, importId, {
+                  processedRows,
+                  successRows,
+                  errorRows,
+                  totalRows,
+                  lastProcessedRow: dataRowIndex,
+                });
+              },
+            });
+
+            logImportExportEvent({
+              entity: 'import',
+              id: importId,
+              subdomain,
+              stage: 'PROCESS_BATCH',
+              event: 'batch_processed',
+              extra: {
+                processedRows,
+                successRows,
+                errorRows,
+                totalRows,
+                lastProcessedRow: dataRowIndex,
+              },
+            });
+
+            if (successIds.length) {
+              await coreClient.addImportedIds(subdomain, importId, successIds);
+            }
+
+            batch = [];
+          }
+
+          if (Object.keys(rowData).length > 0) {
+            batch.push(rowData);
+          } else if (row.some((value) => String(value ?? '').trim() !== '')) {
+            // the row carried values, but none of them sat under a known
+            // column — report it instead of dropping it silently
+            const skippedRow: Record<string, any> = {
+              error:
+                'No value in this row landed in a recognized column. Check the header row.',
+            };
+
+            row.forEach((value: any, index: number) => {
+              const columnName =
+                columnToKeyMap[index] ||
+                headerRow[index] ||
+                `column${index + 1}`;
+              skippedRow[columnName] = value;
+            });
+
+            errorRows++;
+            processedRows++;
+
+            await withImportExportStage({
+              stage: 'WRITE_TEMP_FILE',
+              fallbackMessage: 'Failed to write skipped import row',
+              run: async () => await errorRowWriter.writeRows([skippedRow]),
+            });
+          }
+        }
+
+        if (batch.length > 0) {
+          const result = await withImportExportStage({
+            stage: 'PROCESS_BATCH',
+            fallbackMessage: 'Failed to process final import batch',
+            run: async () =>
+              await config.insertImportRows(
+                {
+                  subdomain,
+                  data: {
+                    moduleName,
+                    collectionName,
+                    rows: batch,
+                    userId,
+                  },
+                },
+                context,
+              ),
+          });
+
+          const successIds =
+            result.successRows
+              ?.map((r: any) => r._id || r.id)
+              .filter(Boolean) || [];
+
+          successRows += successIds.length;
+          errorRows += result.errorRows.length;
+          processedRows += batch.length;
+
+          if (result.errorRows.length) {
+            await withImportExportStage({
+              stage: 'WRITE_TEMP_FILE',
+              fallbackMessage: 'Failed to write final import error rows',
+              run: async () => await errorRowWriter.writeRows(result.errorRows),
+            });
+          }
+
+          await safeProgressUpdate({
+            entity: 'import',
+            id: importId,
+            subdomain,
+            stage: 'final-batch-progress',
+            update: async () => {
+              await coreClient.updateImportProgress(subdomain, importId, {
+                processedRows,
+                successRows,
+                errorRows,
+                totalRows,
+                lastProcessedRow: dataRowIndex,
+              });
+            },
+          });
+
+          logImportExportEvent({
+            entity: 'import',
+            id: importId,
+            subdomain,
+            stage: 'PROCESS_BATCH',
+            event: 'final_batch_processed',
+            extra: {
+              processedRows,
+              successRows,
+              errorRows,
+              totalRows,
+              lastProcessedRow: dataRowIndex,
+            },
+          });
+
+          if (successIds.length) {
+            await coreClient.addImportedIds(subdomain, importId, successIds);
+          }
+        }
+
+        let errorFileUrl: string | null = null;
+        await withImportExportStage({
+          stage: 'FINALIZE_UPLOAD',
+          fallbackMessage: 'Failed to finalize import error file',
+          run: async () => await errorRowWriter.finalize(),
+        });
+
+        if (errorRowWriter.hasErrors()) {
+          errorFileUrl = await withImportExportStage({
+            stage: 'SAVE_RESULT',
+            fallbackMessage: 'Failed to save import error file',
+            run: async () =>
+              await coreClient.saveErrorFile(subdomain, {
+                importId,
+                headerRow,
+                errorRows: errorRowWriter.getErrorRows(),
+                keyToHeaderMap,
+              }),
+          });
+        }
+
+        await coreClient.updateImportProgress(subdomain, importId, {
+          status: 'completed',
+          processedRows,
+          successRows,
+          errorRows,
+          totalRows,
+          lastProcessedRow: dataRowIndex,
+          terminalError: undefined,
+          ...(errorFileUrl && { errorFileUrl }),
+        });
+
+        logImportExportEvent({
+          entity: 'import',
+          id: importId,
+          subdomain,
+          stage: 'COMPLETE',
+          event: 'job_completed',
+          extra: {
+            processedRows,
+            successRows,
+            errorRows,
+            totalRows,
+            lastProcessedRow: dataRowIndex,
+            durationMs: Date.now() - startedAt,
+          },
+        });
+
+        return { success: true };
+      } finally {
+        await safeCleanup({
+          label: `import temp workspace ${importId}`,
+          run: async () => {
+            await tempWorkspace.cleanup();
+          },
+        });
+      }
+    } catch (error: any) {
+      const importExportError =
+        error instanceof ImportExportError ? error : undefined;
+      const terminalError = toTerminalImportExportError(error);
+      const errorCode = `${terminalError.code} @ ${terminalError.stage}`;
+
+      logImportExportEvent({
+        level: 'error',
+        entity: 'import',
+        id: importId,
+        subdomain,
+        stage: terminalError.stage || 'FAILED',
+        event: 'job_failed',
+        extra: {
+          code: terminalError.code,
+          retryable: terminalError.retryable,
+          message: terminalError.message,
+          processedRows,
+          successRows,
+          errorRows,
+          totalRows,
+          lastProcessedRow: dataRowIndex,
+          durationMs: Date.now() - startedAt,
+        },
+      });
+      await coreClient.updateImportProgress(subdomain, importId, {
+        status: 'failed',
+        lastProcessedRow: dataRowIndex,
+        errorMessage: `${errorCode}: ${
+          error?.message || 'Import worker failed'
+        }`,
+        terminalError: {
+          code: terminalError.code,
+          stage: terminalError.stage,
+          retryable: terminalError.retryable,
+        },
+      });
+      if (importExportError && !importExportError.retryable) {
+        throw new UnrecoverableError(importExportError.message);
+      }
+      throw error;
+    }
+  };
+};