@dynamic-entity/server 2.2.0 → 2.3.1

package/express.mjs

@@ -1,4 +1,4 @@
-import { deriveImportColumns, buildTemplateSpec, CORE_VERSION, suggestMapping, toCsv, escapeFormula, validateMappingPlan, applyMapping, createCsvReader, padRow } from '@dynamic-entity/core';
+import { deriveImportColumns, buildTemplateSpec, CORE_VERSION, cellText, arrayBoundFor, suggestMapping, decimalMarkFor, toCsv, escapeFormula, validateMappingPlan, applyMapping, completeFirstLine, padRow, detectDelimiter, createCsvReader } from '@dynamic-entity/core';
 import express from 'express';
 import Busboy from 'busboy';
 import ExcelJS from 'exceljs';
@@ -235,17 +235,6 @@ function readPlanField(fields) {
     throw new ImportError("INVALID_PLAN", "The mapping plan is not valid JSON.", { cause });
   }
 }
-
-// src/cell-text.ts
-function sampleText(raw) {
-  if (raw === null || raw === void 0) return "";
-  if (raw instanceof Date) {
-    if (Number.isNaN(raw.getTime())) return "";
-    const iso = raw.toISOString();
-    return raw.getTime() % 864e5 === 0 ? iso.slice(0, 10) : iso;
-  }
-  return typeof raw === "string" ? raw : String(raw);
-}
 var LOCAL_HEADER = 67324752;
 var CENTRAL_HEADER = 33639248;
 var END_OF_CENTRAL = 101010256;
@@ -562,6 +551,12 @@ async function guardZip(source, limits) {
     }
   }
   if (!sheet) {
+    if (!kept.has("xl/workbook.xml")) {
+      throw new ImportError(
+        "UNSUPPORTED_FORMAT",
+        "This is a zip archive that is not an Excel workbook. Upload an .xlsx or .csv file."
+      );
+    }
     throw new ImportError("MALFORMED_FILE", "This workbook has no worksheet.");
   }
   const ordered = [...WANTED, sheet.name].map((wanted) => kept.get(wanted)).filter((entry) => entry !== void 0);
@@ -656,13 +651,36 @@ function detectFormat(head) {
   }
   return "csv";
 }
-async function* csvRows(stream) {
+async function* csvRows(stream, limits, found, forced) {
   const decoder = new TextDecoder("utf-8");
-  const reader = createCsvReader();
+  const maxHeader = limits.maxCellLength * limits.maxColumns + limits.maxColumns;
+  let held = "";
+  let reader = null;
+  const open = (headerLine) => {
+    found.delimiter = forced ?? detectDelimiter(headerLine);
+    return createCsvReader({ delimiter: found.delimiter });
+  };
   for await (const chunk of stream) {
-    for (const row of reader.push(decoder.decode(chunk, { stream: true }))) yield row;
+    const text = decoder.decode(chunk, { stream: true });
+    if (!reader) {
+      held += text;
+      const line = completeFirstLine(held);
+      if (line === null) {
+        if (held.length > maxHeader) {
+          throw new ImportError("SHEET_TOO_LARGE", "The header row is longer than any row may be.");
+        }
+        continue;
+      }
+      reader = open(line);
+      for (const row of reader.push(held)) yield row;
+      held = "";
+      continue;
+    }
+    for (const row of reader.push(text)) yield row;
   }
-  for (const row of reader.push(decoder.decode())) yield row;
+  const rest = held + decoder.decode();
+  reader ??= open(completeFirstLine(rest) ?? rest);
+  for (const row of reader.push(rest)) yield row;
   for (const row of reader.end()) yield row;
 }
 async function* guardRows(rows, width, limits) {
@@ -697,10 +715,12 @@ async function readSheet(options) {
     const counted = limitBytes(toByteStream(options.stream), limits.maxBytes);
     const { head, stream } = await peek(counted, MAGIC_BYTES);
     const format = detectFormat(head);
-    const rows = format === "xlsx" ? xlsxRows(stream, limits) : csvRows(stream);
+    const found = {};
+    const forced = /\.tsv$/i.test(options.filename ?? "") ? "	" : void 0;
+    const rows = format === "xlsx" ? xlsxRows(stream, limits) : csvRows(stream, limits, found, forced);
     const iterator = rows[Symbol.asyncIterator]();
     const first = await iterator.next();
-    const headers = first.done ? [] : first.value.map(sampleText);
+    const headers = first.done ? [] : first.value.map(cellText);
     if (headers.length > limits.maxColumns) {
       throw new ImportError(
         "SHEET_TOO_LARGE",
@@ -711,7 +731,8 @@ async function readSheet(options) {
     return {
       format,
       headers,
-      rows: closing(guardRows(remaining, headers.length, limits), options.stream)
+      rows: closing(guardRows(remaining, headers.length, limits), options.stream),
+      ...found.delimiter ? { delimiter: found.delimiter } : {}
     };
   } catch (error) {
     destroySource(options.stream);
@@ -765,6 +786,7 @@ async function runImport(options) {
     filename: options.filename,
     limits
   });
+  const decimal = options.decimal ?? decimalMarkFor(sheet.delimiter);
   const errors = [];
   let errorCount = 0;
   let imported = 0;
@@ -780,6 +802,7 @@ async function runImport(options) {
     batch = [];
     const result = applyMapping(rows, plan, config, {
       lang: options.lang,
+      decimal,
       rules: options.rules,
       lookups: options.lookups,
       firstRowNumber: firstRow
@@ -842,19 +865,22 @@ async function previewSheet(options) {
   try {
     for await (const row of sheet.rows) {
       rowCount++;
-      if (sample.length < limits.sampleRows) sample.push(row.map(sampleText));
+      if (sample.length < limits.sampleRows) sample.push(row.map(cellText));
     }
   } catch (error) {
     destroySource(options.stream);
     throw error;
   }
-  const { columns } = deriveImportColumns(options.config, { lang: options.lang });
+  const arrayBound = arrayBoundFor(sheet.headers, options.config, null, options.lang);
+  const { columns } = deriveImportColumns(options.config, { lang: options.lang, maxArrayRows: arrayBound });
   return {
     headers: sheet.headers,
     sample,
     rowCount,
     suggestion: suggestMapping(sheet.headers, columns, options.config?.entity ?? ""),
-    format: sheet.format
+    format: sheet.format,
+    ...sheet.delimiter ? { delimiter: sheet.delimiter } : {},
+    arrayBound
   };
 }
 var TEMPLATE_EXTENSION = { csv: "csv", xlsx: "xlsx" };
@@ -1031,6 +1057,8 @@ function createImportRouter(options) {
         sample: preview.sample,
         suggestion: preview.suggestion,
         rowCount: preview.rowCount,
+        arrayBound: preview.arrayBound,
+        ...preview.delimiter ? { delimiter: preview.delimiter } : {},
         engineVersion: CORE_VERSION
       };
       response.json(body);