@dynamic-entity/server 2.2.0 → 2.3.1

package/README.md

@@ -59,7 +59,7 @@ app.use(
 | Route | Does |
 |---|---|
 | `GET /:entity/template?fields=&format=` | Streams a CSV or xlsx template. No `fields` means every column |
-| `POST /:entity/preview` | multipart → headers, a sample, a suggested mapping, a row count |
+| `POST /:entity/preview` | multipart → headers, a sample, a suggested mapping, a row count, the `arrayBound` the suggestion was sized for, and a CSV's `delimiter` |
 | `POST /:entity/validate` | The identical import pipeline with **no** writing — every error, nothing stored. Answers `written: false` |
 | `POST /:entity/import` | multipart + a `plan` field → runs the import, calling `onImport` per batch. Answers `written: true` |
 
@@ -205,6 +205,12 @@ createImportRouter({ ...options, limits: { maxBytes: 50 * 1024 * 1024, batchSize
 | `maxFields` / `maxFieldBytes` / `maxFieldNameBytes` / `maxFiles` | 16 / 1 MB / 200 / 1 | Multipart abuse |
 | `idleTimeoutMs` / `totalTimeoutMs` | 30 s / 10 min | Stalled and never-ending uploads |
 
+A limit refuses with a code, never a stack: `maxBytes` → `TOO_LARGE` (413); the sheet bounds →
+`SHEET_TOO_LARGE` (413); the three archive bounds → `ARCHIVE_REFUSED` (400), raised while the
+entry inflates, before anything reaches exceljs; the timeouts → `TIMEOUT` (408). A zip that is
+not a workbook at all — a `.docx`, say — is `UNSUPPORTED_FORMAT` (415), as are `.xls` and
+UTF-16 text.
+
 `maxCellLength` is doing double duty: `validators.pattern` is a config-supplied regex that runs
 against cell text, and on a server that text is attacker-chosen. Bounding the input bounds a
 catastrophic backtrack. See [SECURITY.md](../../SECURITY.md).
@@ -263,6 +269,13 @@ Decided by the file's **first bytes**, never by its name — an attacker who wan
 will name their zip `.csv`.
 
 - **CSV** — through core's dependency-free incremental parser. UTF-8, with or without a BOM.
+  The separator — `,`, `;` or tab — is detected from the header line, exactly as the browser
+  detects it, and returned as `delimiter` from a preview. A file named `.tsv` is always split
+  on tabs, as in the browser; that is the one thing a name decides, and only once the bytes
+  have said the file is text. A `;` file is read with a **decimal
+  comma** (`1,5` is one and a half), because Excel writes `;` in precisely the locales that
+  use one; pass `decimal` to `runImport` to override. Generated CSV templates stay
+  comma-separated, so Excel in those locales opens them as one column — offer `xlsx` there.
 - **`.xlsx`** — through `exceljs`. Cells keep their types: a date cell arrives as a `Date` at
   UTC midnight and is read as the calendar date it is, in every timezone.
 - A pre-2007 `.xls` and a UTF-16 file are **named** rather than parsed into nonsense.

package/express.d.mts

@@ -1,6 +1,6 @@
 import { EntityFormConfig, FormRule, ImportLookups } from '@dynamic-entity/core';
 import { Request, Router, Response } from 'express';
-import { B as BatchInfo, I as ImportLimits, T as TemplateFormat } from './write-template-C2dCEp48.mjs';
+import { B as BatchInfo, I as ImportLimits, T as TemplateFormat } from './write-template-CzKVVjgT.mjs';
 import 'node:stream';
 
 /**

package/express.d.ts

@@ -1,6 +1,6 @@
 import { EntityFormConfig, FormRule, ImportLookups } from '@dynamic-entity/core';
 import { Request, Router, Response } from 'express';
-import { B as BatchInfo, I as ImportLimits, T as TemplateFormat } from './write-template-C2dCEp48.js';
+import { B as BatchInfo, I as ImportLimits, T as TemplateFormat } from './write-template-CzKVVjgT.js';
 import 'node:stream';
 
 /**

package/express.js

@@ -244,17 +244,6 @@ function readPlanField(fields) {
     throw new ImportError("INVALID_PLAN", "The mapping plan is not valid JSON.", { cause });
   }
 }
-
-// src/cell-text.ts
-function sampleText(raw) {
-  if (raw === null || raw === void 0) return "";
-  if (raw instanceof Date) {
-    if (Number.isNaN(raw.getTime())) return "";
-    const iso = raw.toISOString();
-    return raw.getTime() % 864e5 === 0 ? iso.slice(0, 10) : iso;
-  }
-  return typeof raw === "string" ? raw : String(raw);
-}
 var LOCAL_HEADER = 67324752;
 var CENTRAL_HEADER = 33639248;
 var END_OF_CENTRAL = 101010256;
@@ -571,6 +560,12 @@ async function guardZip(source, limits) {
     }
   }
   if (!sheet) {
+    if (!kept.has("xl/workbook.xml")) {
+      throw new ImportError(
+        "UNSUPPORTED_FORMAT",
+        "This is a zip archive that is not an Excel workbook. Upload an .xlsx or .csv file."
+      );
+    }
     throw new ImportError("MALFORMED_FILE", "This workbook has no worksheet.");
   }
   const ordered = [...WANTED, sheet.name].map((wanted) => kept.get(wanted)).filter((entry) => entry !== void 0);
@@ -665,13 +660,36 @@ function detectFormat(head) {
   }
   return "csv";
 }
-async function* csvRows(stream) {
+async function* csvRows(stream, limits, found, forced) {
   const decoder = new TextDecoder("utf-8");
-  const reader = core.createCsvReader();
+  const maxHeader = limits.maxCellLength * limits.maxColumns + limits.maxColumns;
+  let held = "";
+  let reader = null;
+  const open = (headerLine) => {
+    found.delimiter = forced ?? core.detectDelimiter(headerLine);
+    return core.createCsvReader({ delimiter: found.delimiter });
+  };
   for await (const chunk of stream) {
-    for (const row of reader.push(decoder.decode(chunk, { stream: true }))) yield row;
+    const text = decoder.decode(chunk, { stream: true });
+    if (!reader) {
+      held += text;
+      const line = core.completeFirstLine(held);
+      if (line === null) {
+        if (held.length > maxHeader) {
+          throw new ImportError("SHEET_TOO_LARGE", "The header row is longer than any row may be.");
+        }
+        continue;
+      }
+      reader = open(line);
+      for (const row of reader.push(held)) yield row;
+      held = "";
+      continue;
+    }
+    for (const row of reader.push(text)) yield row;
   }
-  for (const row of reader.push(decoder.decode())) yield row;
+  const rest = held + decoder.decode();
+  reader ??= open(core.completeFirstLine(rest) ?? rest);
+  for (const row of reader.push(rest)) yield row;
   for (const row of reader.end()) yield row;
 }
 async function* guardRows(rows, width, limits) {
@@ -706,10 +724,12 @@ async function readSheet(options) {
     const counted = limitBytes(toByteStream(options.stream), limits.maxBytes);
     const { head, stream } = await peek(counted, MAGIC_BYTES);
     const format = detectFormat(head);
-    const rows = format === "xlsx" ? xlsxRows(stream, limits) : csvRows(stream);
+    const found = {};
+    const forced = /\.tsv$/i.test(options.filename ?? "") ? "	" : void 0;
+    const rows = format === "xlsx" ? xlsxRows(stream, limits) : csvRows(stream, limits, found, forced);
     const iterator = rows[Symbol.asyncIterator]();
     const first = await iterator.next();
-    const headers = first.done ? [] : first.value.map(sampleText);
+    const headers = first.done ? [] : first.value.map(core.cellText);
     if (headers.length > limits.maxColumns) {
       throw new ImportError(
         "SHEET_TOO_LARGE",
@@ -720,7 +740,8 @@ async function readSheet(options) {
     return {
       format,
       headers,
-      rows: closing(guardRows(remaining, headers.length, limits), options.stream)
+      rows: closing(guardRows(remaining, headers.length, limits), options.stream),
+      ...found.delimiter ? { delimiter: found.delimiter } : {}
     };
   } catch (error) {
     destroySource(options.stream);
@@ -774,6 +795,7 @@ async function runImport(options) {
     filename: options.filename,
     limits
   });
+  const decimal = options.decimal ?? core.decimalMarkFor(sheet.delimiter);
   const errors = [];
   let errorCount = 0;
   let imported = 0;
@@ -789,6 +811,7 @@ async function runImport(options) {
     batch = [];
     const result = core.applyMapping(rows, plan, config, {
       lang: options.lang,
+      decimal,
       rules: options.rules,
       lookups: options.lookups,
       firstRowNumber: firstRow
@@ -851,19 +874,22 @@ async function previewSheet(options) {
   try {
     for await (const row of sheet.rows) {
       rowCount++;
-      if (sample.length < limits.sampleRows) sample.push(row.map(sampleText));
+      if (sample.length < limits.sampleRows) sample.push(row.map(core.cellText));
     }
   } catch (error) {
     destroySource(options.stream);
     throw error;
   }
-  const { columns } = core.deriveImportColumns(options.config, { lang: options.lang });
+  const arrayBound = core.arrayBoundFor(sheet.headers, options.config, null, options.lang);
+  const { columns } = core.deriveImportColumns(options.config, { lang: options.lang, maxArrayRows: arrayBound });
   return {
     headers: sheet.headers,
     sample,
     rowCount,
     suggestion: core.suggestMapping(sheet.headers, columns, options.config?.entity ?? ""),
-    format: sheet.format
+    format: sheet.format,
+    ...sheet.delimiter ? { delimiter: sheet.delimiter } : {},
+    arrayBound
   };
 }
 var TEMPLATE_EXTENSION = { csv: "csv", xlsx: "xlsx" };
@@ -1040,6 +1066,8 @@ function createImportRouter(options) {
         sample: preview.sample,
         suggestion: preview.suggestion,
         rowCount: preview.rowCount,
+        arrayBound: preview.arrayBound,
+        ...preview.delimiter ? { delimiter: preview.delimiter } : {},
         engineVersion: core.CORE_VERSION
       };
       response.json(body);