@rebasepro/admin 0.13.1-canary.gef9608c → 0.14.0

package/src/data_import/components/DataNewPropertiesMapping.tsx

@@ -17,6 +17,7 @@ import {
     Typography
 } from "@rebasepro/ui";
 import { getIn, setIn } from "@rebasepro/forms";
+import { useTranslation } from "@rebasepro/app";
 import { ImportConfig } from "../types";
 
 export interface DataPropertyMappingProps {
@@ -36,6 +37,7 @@ export function DataNewPropertiesMapping({
     buildPropertyView
 }: DataPropertyMappingProps) {
 
+    const { t } = useTranslation();
     const headersMapping = importConfig.headersMapping;
     const headingsOrder = importConfig.headingsOrder;
     const idColumn = importConfig.idColumn;
@@ -117,7 +119,7 @@ export function DataNewPropertiesMapping({
                         <TableCell header={true}>
                         </TableCell>
                         <TableCell header={true} style={{ width: "65%" }}>
-                            Default value
+                            {t("default_value")}
                         </TableCell>
                     </TableHeader>
                     <TableBody>
@@ -205,16 +207,31 @@ function DefaultValuesField({
     onValueChange,
     defaultValue
 }: { property: Property, onValueChange: (value: any) => void, defaultValue?: any }) {
+    const { t } = useTranslation();
+    const defaultValueLabel = t("default_value");
+    // Named fields stay distinguishable from each other — several of these sit
+    // in one table, and an aria-label repeated verbatim tells a screen reader
+    // nothing about which row it is on.
+    //
+    // Composed with a separator rather than a phrase: there is no key for the
+    // possessive form, and `t` returns the key itself for one that does not
+    // exist, so inventing `default_value_for` here would read out the literal
+    // string "default_value_for" in every locale, English included.
+    const ariaLabel = property.name
+        ? `${defaultValueLabel}: ${property.name}`
+        : defaultValueLabel;
     if (property.type === "string") {
         return <TextField size={"medium"}
-            placeholder={"Default value"}
+            aria-label={ariaLabel}
+            placeholder={defaultValueLabel}
             value={defaultValue ?? ""}
             onChange={(event) => onValueChange(event.target.value)}/>;
     } else if (property.type === "number") {
         return <TextField size={"medium"}
             type={"number"}
+            aria-label={ariaLabel}
             value={defaultValue ?? ""}
-            placeholder={"Default value"}
+            placeholder={defaultValueLabel}
             onChange={(event) => onValueChange(event.target.value)}/>;
     } else if (property.type === "boolean") {
         return <BooleanSwitchWithLabel

package/src/data_import/components/ImportNewPropertyFieldPreview.tsx

@@ -38,6 +38,7 @@ export function ImportNewPropertyFieldPreview({
                     {includeName &&
                         <TextField
                             size={"medium"}
+                            aria-label={propertyKey ? `Name of property ${propertyKey}` : "Property name"}
                             className={"text-base grow"}
                             value={property?.name ?? ""}
                             onChange={(e) => {

package/src/data_import/components/ImportSaveInProgress.tsx

@@ -1,10 +1,9 @@
 import { useData } from "@rebasepro/app";
-import { Entity } from "@rebasepro/types";
-import { RebaseData } from "@rebasepro/types";
 import { Button, CenteredView, CircularProgress, Typography } from "@rebasepro/ui";
 import { useEffect, useRef, useState } from "react";
 import { ImportConfig } from "../types";
 import type { AdminCollection } from "@rebasepro/admin-types";
+import { IMPORT_BATCH_SIZE, ImportSaveError, saveImportedEntities } from "../utils/save_entities";
 
 export function ImportSaveInProgress<C extends AdminCollection<any>>
     ({
@@ -20,11 +19,16 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
             onImportSuccess: (collection: C) => void
         }) {
 
-    const [errorSaving, setErrorSaving] = useState<Error | undefined>(undefined);
+    const [errorSaving, setErrorSaving] = useState<ImportSaveError | undefined>(undefined);
     const dataClient = useData();
 
     const savingRef = useRef<boolean>(false);
 
+    // Rows confirmed as written. A retry resumes here rather than re-sending
+    // rows that now exist and conflicting on its first batch, forever.
+    const committedRef = useRef<number>(0);
+    const bulkUnsupportedRef = useRef<{ current: boolean }>({ current: false });
+
     const [processedEntities, setProcessedEntities] = useState<number>(0);
 
     function save() {
@@ -33,20 +37,29 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
             return;
 
         savingRef.current = true;
+        setErrorSaving(undefined);
 
-        saveDataBatch(
+        saveImportedEntities(
             dataClient,
-            collection,
             path,
             importConfig.entities,
-            0,
-            25,
-            setProcessedEntities
+            {
+                offset: committedRef.current,
+                batchSize: IMPORT_BATCH_SIZE,
+                onBatchCommitted: (written) => {
+                    committedRef.current = written;
+                    setProcessedEntities(written);
+                },
+                bulkUnsupported: bulkUnsupportedRef.current
+            }
         ).then(() => {
             onImportSuccess(collection);
             savingRef.current = false;
         }).catch((e) => {
-            setErrorSaving(e);
+            const error = e as ImportSaveError;
+            committedRef.current = error.committed ?? committedRef.current;
+            setProcessedEntities(committedRef.current);
+            setErrorSaving(error);
             savingRef.current = false;
         });
     }
@@ -56,6 +69,10 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
     }, []);
 
     if (errorSaving) {
+        const total = importConfig.entities.length;
+        const failedRows = errorSaving.failedTo - errorSaving.failedFrom > 1
+            ? `rows ${errorSaving.failedFrom + 1}–${errorSaving.failedTo}`
+            : `row ${errorSaving.failedFrom + 1}`;
         return (
             <CenteredView className={"flex flex-col gap-4 items-center"}>
                 <Typography variant={"h6"}>
@@ -65,10 +82,18 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
                 <Typography variant={"body2"} color={"error"}>
                     {errorSaving.message}
                 </Typography>
+
+                <Typography variant={"body2"}>
+                    {errorSaving.committed} of {total} rows were imported.
+                    The import stopped on {failedRows}, which {errorSaving.failedTo - errorSaving.failedFrom > 1 ? "were" : "was"} not written.
+                </Typography>
+
                 <Button
                     onClick={save}
                 >
-                    Retry
+                    {errorSaving.committed > 0
+                        ? `Retry from row ${errorSaving.committed + 1}`
+                        : "Retry"}
                 </Button>
             </CenteredView>
         );
@@ -94,29 +119,3 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
     );
 
 }
-
-function saveDataBatch(dataClient: RebaseData,
-    collection: AdminCollection,
-    path: string,
-    data: Partial<Entity<any>>[],
-    offset = 0,
-    batchSize = 25,
-    onProgressUpdate: (progress: number) => void): Promise<void> {
-
-    console.debug("Saving imported data", offset, batchSize);
-
-    const batch = data.slice(offset, offset + batchSize);
-    return Promise.all(batch.map(d =>
-        dataClient.collection(path).create(
-            d.values,
-            d.id
-        )))
-        .then(() => {
-            if (offset + batchSize < data.length) {
-                onProgressUpdate(offset + batchSize);
-                return saveDataBatch(dataClient, collection, path, data, offset + batchSize, batchSize, onProgressUpdate);
-            }
-            onProgressUpdate(data.length);
-            return Promise.resolve();
-        });
-}

package/src/data_import/import/ImportCollectionAction.tsx

@@ -35,7 +35,7 @@ import { DataNewPropertiesMapping } from "../components/DataNewPropertiesMapping
 import { ImportFileUpload } from "../components/ImportFileUpload";
 import { ImportSaveInProgress } from "../components/ImportSaveInProgress";
 import { ImportConfig } from "../types";
-import { slugify } from "@rebasepro/utils";
+import { isPrototypePollutingKey, slugify } from "@rebasepro/utils";
 
 type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
 
@@ -400,7 +400,15 @@ export function ImportDataPreview<M extends Record<string, unknown>>({
     return <CollectionTableBinding
         title={<div>
             <Typography variant={"subtitle2"}>Imported data preview</Typography>
-            <Typography variant={"caption"}>Entities with the same id will be overwritten</Typography>
+            {/* Conditional because it is only true when a column was chosen as
+                the id: without one, every row is created and nothing can be
+                overwritten. With one, the import asks for an upsert — which it
+                did not, back when this sentence was unconditional and false. */}
+            <Typography variant={"caption"}>
+                {importConfig.idColumn
+                    ? "Entities with the same id will be overwritten"
+                    : "All rows will be imported as new entities"}
+            </Typography>
         </div>}
         tableController={{
             data: importConfig.entities,
@@ -421,6 +429,9 @@ function buildHeadersMappingFromData(objArr: object[], properties?: Properties)
     const headersMapping: Record<string, string> = {};
     objArr.filter(Boolean).forEach((obj) => {
         Object.keys(obj).forEach((key) => {
+            // The keys are the uploaded file's header row; `headersMapping[key] = …`
+            // is the same setter the import pipeline refuses elsewhere.
+            if (isPrototypePollutingKey(key)) return;
             const child = (obj as Record<string, unknown>)[key];
             if (child != null && typeof child === "object" && !Array.isArray(child)) {
                 const childProperty = properties?.[key];

package/src/data_import/utils/csv.ts

@@ -0,0 +1,153 @@
+import { isPrototypePollutingKey } from "@rebasepro/utils";
+
+/**
+ * Delimiters sniffed from the header row, most common first.
+ *
+ * A spreadsheet saved as "CSV" in a locale that uses the comma as a decimal
+ * separator is semicolon-delimited, and a tab-separated file saved as `.csv`
+ * happens often enough to be worth reading.
+ */
+const CANDIDATE_DELIMITERS = [",", ";", "\t"];
+
+/**
+ * Split CSV text into rows of raw cells, RFC 4180 style.
+ *
+ * Quoted fields may contain the delimiter, newlines and doubled quotes; this is
+ * the exact inverse of what the export's `entryToCSVRow` writes, so a file this
+ * admin produced reads back into the cells it was built from.
+ */
+export function parseCsvRows(text: string, delimiter = ","): string[][] {
+    const rows: string[][] = [];
+    let row: string[] = [];
+    let field = "";
+    let quoted = false;
+    let inQuotes = false;
+    let i = 0;
+
+    const endField = () => {
+        row.push(field);
+        field = "";
+        quoted = false;
+    };
+
+    while (i < text.length) {
+        const char = text[i];
+
+        if (inQuotes) {
+            if (char === "\"") {
+                // A doubled quote is a literal one; a single quote closes the field.
+                if (text[i + 1] === "\"") {
+                    field += "\"";
+                    i += 2;
+                    continue;
+                }
+                inQuotes = false;
+                i++;
+                continue;
+            }
+            field += char;
+            i++;
+            continue;
+        }
+
+        if (char === "\"" && field === "") {
+            inQuotes = true;
+            quoted = true;
+            i++;
+            continue;
+        }
+        if (char === delimiter) {
+            endField();
+            i++;
+            continue;
+        }
+        if (char === "\r") {
+            // CRLF: the line ends on the \n. A lone CR outside quotes is dropped.
+            i++;
+            continue;
+        }
+        if (char === "\n") {
+            endField();
+            rows.push(row);
+            row = [];
+            i++;
+            continue;
+        }
+        field += char;
+        i++;
+    }
+
+    // A file that does not end in a newline still has a last row.
+    if (field !== "" || quoted || row.length > 0) {
+        endField();
+        rows.push(row);
+    }
+
+    return rows;
+}
+
+/**
+ * Pick the delimiter that splits the header row into the most cells. A
+ * single-column file matches nothing and stays on the comma.
+ */
+export function detectCsvDelimiter(text: string): string {
+    const sample = text.slice(0, 64 * 1024);
+    let best = CANDIDATE_DELIMITERS[0];
+    let bestCount = 1;
+    for (const candidate of CANDIDATE_DELIMITERS) {
+        const count = parseCsvRows(sample, candidate)[0]?.length ?? 0;
+        if (count > bestCount) {
+            best = candidate;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+
+export interface ParsedCsv {
+    headers: string[];
+    data: Record<string, string>[];
+}
+
+/**
+ * Parse CSV text into one object per row, keyed by the header row.
+ *
+ * The picker accepts `.csv` and the dialog promises "Upload a CSV, Excel or JSON
+ * file", but every non-JSON file used to be handed to ExcelJS'
+ * `workbook.xlsx.load`, which is a zip reader — so every CSV failed with
+ * "Can't find end of central directory : is this a zip file ?". The export
+ * writes CSV by default, so the round trip could not close.
+ *
+ * A blank header keeps its column rather than collapsing it, so the cells to its
+ * right stay under their own names.
+ */
+export function parseCsvToObjects(text: string): ParsedCsv {
+    // A BOM is one character of the first header, and Excel writes one.
+    const cleaned = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text;
+    const delimiter = detectCsvDelimiter(cleaned);
+    const rows = parseCsvRows(cleaned, delimiter)
+        .filter(row => row.some(cell => cell.trim() !== ""));
+
+    if (rows.length === 0) {
+        return { headers: [],
+            data: [] };
+    }
+
+    const headers = rows[0].map((header, index) => header.trim() || `Column${index + 1}`);
+
+    const data = rows.slice(1).map((cells) => {
+        const obj: Record<string, string> = {};
+        headers.forEach((header, index) => {
+            // The header row is uploaded data: `obj["__proto__"] = …` is the
+            // prototype setter, not a column. Refused, as elsewhere in import.
+            if (isPrototypePollutingKey(header)) return;
+            const cell = cells[index];
+            if (cell === undefined) return;
+            obj[header] = cell;
+        });
+        return obj;
+    });
+
+    return { headers,
+        data };
+}

package/src/data_import/utils/data.ts

@@ -5,7 +5,7 @@ import { isPropertyBuilder } from "@rebasepro/common";
 import { unflattenObject } from "./file_to_json";
 import { getIn } from "@rebasepro/forms";
 import { inferTypeFromValue } from "@rebasepro/inference";
-import { mergeDeep } from "@rebasepro/utils";
+import { isPrototypePollutingKey, mergeDeep } from "@rebasepro/utils";
 
 export function convertDataToEntity(authController: AuthController,
     navigation: CollectionRegistryController,
@@ -57,6 +57,12 @@ export function convertDataToEntity(authController: AuthController,
 
 export function flattenEntry(obj: Record<string, unknown>, parent = ""): Record<string, unknown> {
     return Object.keys(obj).reduce<Record<string, unknown>>((acc, key) => {
+        // Keys come from the uploaded file, so `acc[key] = …` is the class-22
+        // primitive when the key is `__proto__`; refuse the column instead.
+        if (isPrototypePollutingKey(key)) {
+            console.warn(`Skipping column "${key}": a header may not reach the prototype chain`);
+            return acc;
+        }
         const prefixedKey = parent ? `${parent}.${key}` : key;
 
         if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
@@ -100,7 +106,15 @@ export function processValueMapping(authController: AuthController, value: any,
     if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !Array.isArray(usedProperty.of) && !isPropertyBuilder(usedProperty.of)) {
         return value.map(v => processValueMapping(authController, v, navigation, usedProperty.of as Property));
     } else if (from === "string" && to === "number" && typeof value === "string") {
-        return Number(value);
+        // `Number("")` is 0, so a blank cell used to import as a real zero — a
+        // price of nothing rather than a price nobody filled in — and `Number`
+        // of anything unreadable imported as NaN. Both are absent values, and
+        // saying so lets the importer's own validation see them. The `vector`
+        // branch above already draws the same distinction.
+        const trimmed = value.trim();
+        if (trimmed === "") return null;
+        const num = Number(trimmed);
+        return Number.isNaN(num) ? null : num;
     } else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !Array.isArray(usedProperty.of) && !isPropertyBuilder(usedProperty.of)) {
         return value.split(",").map((v: string) => processValueMapping(authController, v, navigation, usedProperty.of as Property));
     } else if (from === "string" && to === "boolean") {

package/src/data_import/utils/file_to_json.ts

@@ -1,6 +1,7 @@
-import ExcelJS from "exceljs";
 import { getWorksheetHeaders } from "./file_headers";
 import { mapJsonParse, unflattenObject } from "./transforms";
+import { parseCsvToObjects } from "./csv";
+import { isPrototypePollutingKey } from "@rebasepro/utils";
 
 // Re-export for backwards compat (tests import from this file)
 export { unflattenObject } from "./transforms";
@@ -10,9 +11,78 @@ type ConversionResult = {
     propertiesOrder: string[]
 }
 
+/**
+ * ExcelJS, fetched the first time somebody actually opens a workbook.
+ *
+ * It was a top-level `import ExcelJS from "exceljs"`, and that one line put
+ * 940 kB of spreadsheet reader into the admin's entry chunk — preloaded on the
+ * login screen, before anyone has authenticated, let alone clicked Import.
+ * `CollectionViewActions` lazy-loads the import and export actions and says so
+ * in a comment, but the package's own barrel (`src/index.ts` re-exports
+ * `./data_import`) puts this module back in the entry's static graph, so the
+ * `lazy()` bought nothing. A static import inside a module the barrel reaches
+ * is eager no matter what the component above it does; the only thing that
+ * makes a dependency lazy is importing it lazily, here.
+ */
+type ExcelJsModule = typeof import("exceljs");
+
+let excelJsModule: Promise<ExcelJsModule> | undefined;
+
+function loadExcelJs(): Promise<ExcelJsModule> {
+    // exceljs is CommonJS, so the namespace a bundler hands back wraps the real
+    // module under `default`. Native ESM (and the type declarations) put the
+    // members at the top level. Accept either.
+    excelJsModule ??= import("exceljs").then(mod => (mod as { default?: ExcelJsModule }).default ?? mod);
+    return excelJsModule;
+}
+
+/**
+ * Whether this file is delimited text rather than a workbook.
+ *
+ * Browsers report `text/csv`, `application/csv` or nothing at all for the same
+ * `.csv`, so the extension is the reliable half of the test.
+ */
+function isCsvFile(file: File): boolean {
+    const name = (file.name ?? "").toLowerCase();
+    if (name.endsWith(".csv") || name.endsWith(".tsv")) return true;
+    return file.type === "text/csv" || file.type === "application/csv";
+}
+
+/** Turn the parsed cells of one sheet/file into the entity-shaped rows the import maps. */
+function toImportRows(rows: Array<Record<string, unknown>>): object[] {
+    return rows.map(mapJsonParse).map(unflattenObject);
+}
+
 export function convertFileToJson(file: File): Promise<ConversionResult> {
     return new Promise((resolve, reject) => {
-        if (file.type === "application/json") {
+        if (isCsvFile(file)) {
+            // Not the ExcelJS branch: `workbook.xlsx.load` is a zip reader, and
+            // every CSV — the format this admin exports by default — died on it
+            // with "Can't find end of central directory : is this a zip file ?".
+            console.debug("Converting CSV file to JSON", file.name);
+            const reader = new FileReader();
+            reader.onload = function (e) {
+                try {
+                    const { headers, data } = parseCsvToObjects(e.target?.result as string);
+                    if (headers.length === 0) {
+                        reject(new Error("The CSV file is empty"));
+                        return;
+                    }
+                    resolve({
+                        data: toImportRows(data),
+                        propertiesOrder: headers
+                    });
+                } catch (err) {
+                    console.error("Error parsing CSV file", err);
+                    reject(err);
+                }
+            };
+            reader.onerror = () => reject(reader.error ?? new Error("Could not read the file"));
+            // Explicit UTF-8: the pre-ExcelJS implementation passed
+            // `codepage: 65001` for the same reason, and the browser's default
+            // guess mangles accented and CJK text.
+            reader.readAsText(file, "utf-8");
+        } else if (file.type === "application/json") {
             console.debug("Converting JSON file to JSON", file.name);
             const reader = new FileReader();
             reader.onload = function (e) {
@@ -40,6 +110,7 @@ export function convertFileToJson(file: File): Promise<ConversionResult> {
             reader.onload = async function (e) {
                 try {
                     const buffer = e.target?.result as ArrayBuffer;
+                    const ExcelJS = await loadExcelJs();
                     const workbook = new ExcelJS.Workbook();
                     await workbook.xlsx.load(buffer);
                     const worksheet = workbook.worksheets[0];
@@ -57,17 +128,17 @@ export function convertFileToJson(file: File): Promise<ConversionResult> {
                         const obj: Record<string, any> = {};
                         row.eachCell({ includeEmpty: false }, (cell, colNumber) => {
                             const header = headers[colNumber - 1];
-                            if (header) {
+                            // A `__proto__` header would be the prototype setter
+                            // here rather than a column; refused, as in `csv.ts`.
+                            if (header && !isPrototypePollutingKey(header)) {
                                 obj[header] = cell.value;
                             }
                         });
                         parsedData.push(obj);
                     });
 
-                    const cleanedData = parsedData.map(mapJsonParse);
-                    const jsonData = cleanedData.map(unflattenObject);
                     resolve({
-                        data: jsonData,
+                        data: toImportRows(parsedData),
                         propertiesOrder: headers
                     });
                 } catch (err) {

package/src/data_import/utils/index.ts

@@ -1,3 +1,5 @@
 export * from "./file_to_json";
+export * from "./csv";
+export * from "./save_entities";
 export * from "./data";
 export * from "./get_import_inference_type";