@rebasepro/cms 0.17.3 → 0.18.0

package/src/data_export/export/export.ts

@@ -1,295 +0,0 @@
-import { Entity, EntityReference, Properties, Property } from "@rebasepro/types";
-import { type ArrayValuesCount, getArrayValuesCount, getValueInPath } from "@rebasepro/utils";
-
-interface Header {
-    key: string;
-    label: string;
-}
-
-export interface DownloadEntitiesExportParams<M extends Record<string, unknown>> {
-    data: Entity<M>[];
-    additionalData: Record<string, unknown>[] | undefined;
-    properties: Properties;
-    propertiesOrder: string[] | undefined;
-    name: string;
-    flattenArrays: boolean;
-    additionalHeaders: string[] | undefined;
-    exportType: "csv" | "json";
-    dateExportType: "timestamp" | "string";
-}
-
-export function downloadEntitiesExport<M extends Record<string, unknown>>({
-    data,
-    additionalData,
-    properties,
-    propertiesOrder,
-    name,
-    flattenArrays,
-    additionalHeaders,
-    exportType,
-    dateExportType
-}: DownloadEntitiesExportParams<M>
-) {
-
-    console.debug("Downloading export", {
-        dataLength: data.length,
-        properties,
-        exportType,
-        dateExportType
-    });
-
-    if (exportType === "csv") {
-        const arrayValuesCount = flattenArrays ? getArrayValuesCount(data.map(d => d.values)) : {};
-        const headers = getExportHeaders(properties, propertiesOrder, additionalHeaders, arrayValuesCount);
-        const exportableData = getEntityCSVExportableData(data, additionalData, properties, headers, dateExportType);
-        const headersData = entryToCSVRow(headers.map(h => h.label));
-        const csvData = exportableData.map(entry => entryToCSVRow(entry));
-        downloadBlob([headersData, ...csvData], `${name}.csv`, "text/csv");
-    } else {
-        const exportableData = getEntityJsonExportableData(data, additionalData, properties, dateExportType);
-        const json = JSON.stringify(exportableData, null, 2);
-        downloadBlob([json], `${name}.json`, "application/json");
-    }
-}
-
-export function getEntityCSVExportableData(data: Entity<Record<string, unknown>>[],
-    additionalData: Record<string, unknown>[] | undefined,
-    properties: Properties,
-    headers: Header[],
-    dateExportType: "timestamp" | "string"
-) {
-
-    const mergedData: Record<string, unknown>[] = data.map(e => ({
-        id: e.id,
-        ...processValuesForExport(e.values, properties, "csv", dateExportType)
-    }));
-
-    if (additionalData) {
-        additionalData.forEach((additional, index) => {
-            mergedData[index] = { ...mergedData[index],
-...additional };
-        });
-    }
-
-    return mergedData && mergedData.map((entry) => {
-        return headers.map((header) => getValueInPath(entry, header.key));
-    });
-}
-
-export function getEntityJsonExportableData(data: Entity<Record<string, unknown>>[],
-    additionalData: Record<string, unknown>[] | undefined,
-    properties: Properties,
-    dateExportType: "timestamp" | "string"
-) {
-
-    const mergedData: Record<string, unknown>[] = data.map(e => ({
-        id: e.id,
-        ...processValuesForExport(e.values, properties, "json", dateExportType)
-    }));
-
-    if (additionalData) {
-        additionalData.forEach((additional, index) => {
-            mergedData[index] = { ...mergedData[index],
-...additional };
-        });
-    }
-
-    return mergedData;
-}
-
-function getExportHeaders<M extends Record<string, unknown>>(properties: Properties,
-    propertiesOrder: string[] | undefined,
-    additionalHeaders: string[] | undefined,
-    arrayValuesCount?: ArrayValuesCount): Header[] {
-
-    const headers: Header[] = [
-        {
-            label: "id",
-            key: "id"
-        },
-        ...(propertiesOrder ?? Object.keys(properties))
-            .flatMap((childKey) => {
-                const property = properties[childKey];
-                if (!property) {
-                    console.warn("Property not found", childKey, properties);
-                    return [];
-                }
-                if (arrayValuesCount && arrayValuesCount[childKey] > 1) {
-                    return Array.from({ length: arrayValuesCount[childKey] },
-                        (_, i) => getHeaders(property as Property, `${childKey}[${i}]`, ""))
-                        .flat();
-                } else {
-                    return getHeaders(property as Property, childKey, "");
-                }
-            })
-    ];
-
-    if (additionalHeaders) {
-        headers.push(...additionalHeaders.map(h => ({
-            label: h,
-            key: h
-        })));
-    }
-
-    return headers;
-}
-
-/**
- * Get headers for property. There could be more than one header per property
- * @param property
- * @param propertyKey
- * @param prefix
- */
-function getHeaders(property: Property, propertyKey: string, prefix = ""): Header[] {
-    const currentKey = prefix ? `${prefix}.${propertyKey}` : propertyKey;
-    if (property.type === "map" && property.properties) {
-        return Object.entries(property.properties)
-            .map(([childKey, p]) => getHeaders(p, childKey, currentKey))
-            .flat();
-    } else {
-        return [{
-            label: currentKey,
-            key: currentKey
-        }];
-    }
-}
-
-/**
- * Read whatever a `date` column actually arrived as into a `Date`.
- *
- * The driver/WebSocket path revives `{ __type: "date" }` into a `Date`; the REST
- * path — which is the one the admin is on — leaves an ISO string, and an epoch
- * number is what a re-imported export carries. Returns `undefined` for anything
- * that is not a date, so the caller can pass the raw value through.
- */
-function toExportableDate(inputValue: unknown): Date | undefined {
-    if (inputValue instanceof Date) return Number.isNaN(inputValue.getTime()) ? undefined : inputValue;
-    if (typeof inputValue === "number" && Number.isFinite(inputValue)) return new Date(inputValue);
-    if (typeof inputValue === "string" && inputValue.trim() !== "") {
-        const parsed = new Date(inputValue);
-        return Number.isNaN(parsed.getTime()) ? undefined : parsed;
-    }
-    return undefined;
-}
-
-function processValueForExport(inputValue: unknown,
-    property: Property,
-    exportType: "csv" | "json",
-    dateExportType: "timestamp" | "string"
-): unknown {
-
-    let value: unknown;
-    if (property.type === "map" && property.properties) {
-        value = processValuesForExport(inputValue as Record<string, unknown>, property.properties as Properties, exportType, dateExportType);
-    } else if (property.type === "array") {
-        if (property.of && Array.isArray(inputValue)) {
-            if (Array.isArray(property.of)) {
-                value = property.of.map((p, i) => processValueForExport(inputValue[i], p, exportType, dateExportType));
-            } else if (property.of.type === "map") {
-                value = exportType === "csv"
-                    ? inputValue.map((e) => JSON.stringify(e))
-                    : inputValue.map((e) => processValueForExport(e, property.of as Property, exportType, dateExportType));
-                ;
-            } else {
-                value = inputValue.map((e) => processValueForExport(e, property.of as Property, exportType, dateExportType));
-            }
-        } else {
-            value = inputValue;
-        }
-    } else if (property.type === "reference" && inputValue && typeof inputValue === "object" && "isEntityReference" in inputValue && typeof (inputValue as EntityReference).isEntityReference === "function" && (inputValue as EntityReference).isEntityReference()) {
-        const ref = inputValue ? inputValue as EntityReference : undefined;
-        value = ref ? ref.fullPath : null;
-    } else if (property.type === "date") {
-        // Not `instanceof Date`: the admin reads over REST, and the REST row
-        // pipeline hands dates back as the database wrote them — a string. That
-        // made this branch dead on every live export, so both radio options
-        // produced identical files. Normalise here, where the declared type says
-        // the column is a date, and leave anything unparseable untouched rather
-        // than exporting an invalid date as `null`.
-        const date = toExportableDate(inputValue);
-        value = date
-            ? (dateExportType === "timestamp" ? date.getTime() : date.toISOString())
-            : inputValue;
-    } else {
-        value = inputValue;
-    }
-
-    return value;
-}
-
-function processValuesForExport<M extends Record<string, unknown>>
-    (inputValues: Record<string, unknown>,
-        properties: Properties,
-        exportType: "csv" | "json",
-        dateExportType: "timestamp" | "string"
-    ): Record<string, unknown> {
-    const updatedValues = Object.entries(properties)
-        .map(([key, property]) => {
-            const inputValue = inputValues && (inputValues)[key];
-            const updatedValue = processValueForExport(inputValue, property as Property, exportType, dateExportType);
-            if (updatedValue === undefined) return {};
-            return ({ [key]: updatedValue });
-        })
-        .reduce((a, b) => ({ ...a,
-...b }), {}) as Record<string, unknown>;
-    return { ...inputValues,
-...updatedValues };
-}
-
-/**
- * Characters that make a spreadsheet read a cell as a formula rather than text.
- *
- * Quoting is CSV *escaping* and is not enough: Excel, LibreOffice Calc and
- * Sheets strip the CSV quoting at parse time and then evaluate a cell whose
- * first character is one of these — `=HYPERLINK(…)` exfiltrates neighbouring
- * cells on click, `=cmd|'…'!A0` is worse.
- */
-const FORMULA_TRIGGERS = ["=", "+", "-", "@", "\t", "\r"];
-
-/**
- * Neutralise a cell that a spreadsheet would otherwise evaluate, by prefixing a
- * single quote — the standard OWASP mitigation. The apostrophe is consumed by
- * the spreadsheet as "treat the rest as text", so the value reads back intact.
- */
-export function escapeCsvFormula(value: string): string {
-    const first = value.charAt(0);
-    if (!FORMULA_TRIGGERS.includes(first)) return value;
-    // A signed number is not a formula, and prefixing it would turn every
-    // negative amount in the file into text. `-1+1` is still neutralised,
-    // because it is not a number.
-    if ((first === "-" || first === "+") && value.trim() !== "" && Number.isFinite(Number(value))) return value;
-    return "'" + value;
-}
-
-function toCSVCell(v: unknown): string {
-    if (v === null || v === undefined) return "";
-    const s = Array.isArray(v) ? JSON.stringify(v) : String(v);
-    return "\"" + escapeCsvFormula(s).replaceAll("\"", "\"\"") + "\"";
-}
-
-export function entryToCSVRow(entry: unknown[]) {
-    return entry
-        .map(toCSVCell)
-        .join(",") + "\r\n";
-}
-
-export function downloadBlob(content: BlobPart[], filename: string, contentType: string) {
-    const blob = new Blob(content, { type: contentType });
-    const url = URL.createObjectURL(blob);
-    const pom = document.createElement("a");
-    pom.href = url;
-    pom.setAttribute("download", filename);
-    pom.click();
-}
-
-export function downloadDataAsCsv(data: object[], name: string) {
-    if (data.length === 0) return;
-
-    const headers = Object.keys(data[0]);
-    const csvContent = [
-        headers.join(","),
-        ...data.map(row => headers.map(header => toCSVCell((row as Record<string, unknown>)[header])).join(","))
-    ].join("\r\n");
-
-    downloadBlob([csvContent], `${name}.csv`, "text/csv");
-}

package/src/data_export/export/fetch_export_data.ts

@@ -1,90 +0,0 @@
-import { Entity, FindParams, FindResponse } from "@rebasepro/types";
-
-/**
- * Rows asked for per request.
- *
- * Kept at or below the server's `MAX_LIST_LIMIT`, which is now the largest page
- * a read will serve *and refuses* anything larger — it used to clamp silently,
- * and asking for 100 000 while receiving 1 000 looked exactly like a collection
- * with 1 000 rows in it. The walk still never trusts the page size it asked for:
- * it advances by the rows it was actually handed.
- */
-export const EXPORT_PAGE_SIZE = 500;
-
-/**
- * Most rows one browser-side export will materialise.
- *
- * Everything here is built in memory and handed to a Blob, so there is a
- * ceiling whether or not it is named. Naming it means going over it **throws**
- * instead of writing a short file that reads like the whole collection.
- */
-export const MAX_EXPORT_ROWS = 100_000;
-
-export interface FetchAllForExportParams<M extends Record<string, unknown>> {
-    /** The collection accessor to read through — `dataClient.collection(path)`. */
-    accessor: { find(params?: FindParams<M>): Promise<FindResponse<M>> };
-    /** Query to export. The window (`limit` / `offset` / `page`) is owned by the walk. */
-    params?: Omit<FindParams<M>, "limit" | "offset" | "page">;
-    pageSize?: number;
-    maxRows?: number;
-    /** Called after every page with the rows so far, and the total when the server reports one. */
-    onProgress?: (loaded: number, total: number | undefined) => void;
-}
-
-/**
- * Read **every** row matching a query, one page at a time.
- *
- * `find({})` sends no `limit`, and an absent limit is resolved server-side to
- * `DEFAULT_LIST_LIMIT` — 50 rows. The export used to hand those 50 rows back as
- * `<collection>.csv` with nothing saying it was a sample, next to a dialog
- * warning that the collection had 100 000 documents.
- *
- * Termination is driven by the server's `meta.hasMore` where it is reported and
- * by an empty page otherwise — never by comparing a page's length against the
- * requested limit, since a final page that happens to be exactly full is
- * indistinguishable that way and a walk that stops there drops rows.
- *
- * @throws When more than `maxRows` rows match. A truncated export is worse than
- * no export: it is indistinguishable from a complete one.
- */
-export async function fetchAllEntitiesForExport<M extends Record<string, unknown>>({
-    accessor,
-    params,
-    pageSize = EXPORT_PAGE_SIZE,
-    maxRows = MAX_EXPORT_ROWS,
-    onProgress
-}: FetchAllForExportParams<M>): Promise<Entity<M>[]> {
-
-    const rows: Entity<M>[] = [];
-    let offset = 0;
-
-    for (;;) {
-        const res = await accessor.find({
-            ...params,
-            limit: pageSize,
-            offset
-        } as FindParams<M>);
-
-        const page = res?.data ?? [];
-        rows.push(...page);
-        onProgress?.(rows.length, res?.meta?.total);
-
-        if (rows.length > maxRows) {
-            throw new Error(
-                `This export would contain more than ${maxRows.toLocaleString()} rows, which is more than the ` +
-                "browser can build in one file. Filter the collection down, or export it from the server."
-            );
-        }
-
-        // An empty page ends the walk whatever the metadata says, so a server
-        // that keeps answering `hasMore` cannot spin forever.
-        if (page.length === 0) break;
-        if (res?.meta && !res.meta.hasMore) break;
-
-        // By rows received, not by `pageSize`: a server whose ceiling is lower
-        // than the page we asked for would otherwise skip everything it clamped.
-        offset += page.length;
-    }
-
-    return rows;
-}

package/src/data_export/export/index.ts

@@ -1,4 +0,0 @@
-export * from "./export";
-export * from "./fetch_export_data";
-export * from "./BasicExportAction";
-export * from "./ExportCollectionAction";

package/src/data_export/index.ts

@@ -1 +0,0 @@
-export * from "./export";

package/src/data_import/components/DataNewPropertiesMapping.tsx

@@ -1,261 +0,0 @@
-
-import { getPropertyInPath } from "../../util";
-import { Properties, Property } from "@rebasepro/types";
-import {
-    BooleanSwitchWithLabel,
-    ChevronRightIcon,
-    DateTimeField,
-    ExpandablePanel,
-    Select,
-    SelectItem,
-    Table,
-    TableBody,
-    TableCell,
-    TableHeader,
-    TableRow,
-    TextField,
-    Typography
-} from "@rebasepro/ui";
-import { getIn, setIn } from "@rebasepro/forms";
-import { useTranslation } from "@rebasepro/app";
-import { ImportConfig } from "../types";
-
-export interface DataPropertyMappingProps {
-    importConfig: ImportConfig;
-    destinationProperties: Record<string, Property>;
-    buildPropertyView?: (props: {
-        isIdColumn: boolean,
-        property: Property | null,
-        propertyKey: string | null,
-        importKey: string
-    }) => any;
-}
-
-export function DataNewPropertiesMapping({
-    importConfig,
-    destinationProperties,
-    buildPropertyView
-}: DataPropertyMappingProps) {
-
-    const { t } = useTranslation();
-    const headersMapping = importConfig.headersMapping;
-    const headingsOrder = importConfig.headingsOrder;
-    const idColumn = importConfig.idColumn;
-    const originProperties = importConfig.originProperties;
-
-    return (
-        <>
-
-            <IdSelectField idColumn={idColumn}
-                headersMapping={headersMapping}
-                onChange={(value) => importConfig.setIdColumn(value ?? undefined)}/>
-
-            <div className={"h-4"}/>
-
-            <Table style={{
-                tableLayout: "fixed"
-            }}>
-                <TableHeader>
-                    <TableCell header={true} style={{ width: "20%" }}>
-                        Column in file
-                    </TableCell>
-                    <TableCell header={true}>
-                    </TableCell>
-                    <TableCell header={true} style={{ width: "75%" }}>
-                        Map to Property
-                    </TableCell>
-                </TableHeader>
-                <TableBody>
-                    {destinationProperties &&
-                        headingsOrder.map((importKey) => {
-                            const mappedKey = headersMapping[importKey];
-                            const propertyKey = headersMapping[importKey];
-                            const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
-
-                            const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
-                            const origintype = originProperty
-                                ? (originProperty.type === "array" && typeof originProperty.of === "object"
-                                    ? `${originProperty.type} - ${(originProperty.of as Property).type}`
-                                    : originProperty.type)
-                                : undefined;
-                            return <TableRow key={importKey} style={{ height: "90px" }}>
-                                <TableCell style={{ width: "20%" }}>
-                                    <Typography variant={"body2"}>{importKey}</Typography>
-                                    {originProperty && <Typography
-                                        variant={"caption"}
-                                        color={"secondary"}
-                                    >{origintype}</Typography>}
-                                </TableCell>
-                                <TableCell>
-                                    <ChevronRightIcon/>
-                                </TableCell>
-                                <TableCell className={importKey === idColumn ? "text-center" : undefined}
-                                    style={{ width: "75%" }}>
-                                    {buildPropertyView?.({
-                                        isIdColumn: importKey === idColumn,
-                                        property,
-                                        propertyKey,
-                                        importKey
-                                    })}
-                                </TableCell>
-                            </TableRow>;
-                        }
-                        )}
-                </TableBody>
-            </Table>
-
-            <ExpandablePanel title="Default values" initiallyExpanded={false} innerClassName={"p-4 mt-4"}>
-
-                <div className={"text-sm text-surface-accent-500 dark:text-surface-accent-300 font-medium ml-3.5 mb-1"}>
-                    You can select a default value for unmapped columns and empty values:
-                </div>
-                <Table style={{
-                    tableLayout: "fixed"
-                }}>
-                    <TableHeader>
-                        <TableCell header={true} style={{ width: "30%" }}>
-                            Property
-                        </TableCell>
-                        <TableCell header={true}>
-                        </TableCell>
-                        <TableCell header={true} style={{ width: "65%" }}>
-                            {t("default_value")}
-                        </TableCell>
-                    </TableHeader>
-                    <TableBody>
-                        {destinationProperties &&
-                            getAllPropertyKeys(destinationProperties).map((key) => {
-                                const property = getPropertyInPath(destinationProperties, key);
-                                if (typeof property !== "object" || property === null) {
-                                    return null;
-                                }
-                                if (!["number", "string", "boolean", "map"].includes(property.type)) {
-                                    return null;
-                                }
-                                return <TableRow key={key} style={{ height: "70px" }}>
-                                    <TableCell style={{ width: "20%" }}>
-                                        <Typography variant={"body2"}>{key}</Typography>
-                                    </TableCell>
-                                    <TableCell>
-                                        <ChevronRightIcon/>
-                                    </TableCell>
-                                    <TableCell className={key === idColumn ? "text-center" : undefined}
-                                        style={{ width: "75%" }}>
-                                        <DefaultValuesField property={property}
-                                            defaultValue={getIn(importConfig.defaultValues, key)}
-                                            onValueChange={(value) => {
-                                                const newValues = setIn(importConfig.defaultValues, key, value);
-                                                importConfig.setDefaultValues(newValues as Record<string, any>);
-                                            }}/>
-                                    </TableCell>
-                                </TableRow>;
-                            }
-                            )}
-                    </TableBody>
-                </Table>
-            </ExpandablePanel>
-        </>
-    );
-}
-
-function getAllPropertyKeys(properties: Properties, currentKey?: string): string[] {
-    return Object.entries(properties).reduce((acc, [key, property]) => {
-        const accumulatedKey = currentKey ? `${currentKey}.${key}` : key;
-        if (typeof property !== "function" && property.type === "map" && property.properties) {
-            const childProperties = getAllPropertyKeys(property.properties, accumulatedKey);
-            return [...acc, ...childProperties];
-        }
-        return [...acc, accumulatedKey];
-    }, [] as string[]);
-}
-
-function IdSelectField({
-    idColumn,
-    headersMapping,
-    onChange
-}: {
-    idColumn?: string,
-    headersMapping: Record<string, string | null>;
-    onChange: (value: string | null) => void
-}) {
-    return <div>
-        <Select
-            size={"medium"}
-            value={idColumn ?? ""}
-            fullWidth={true}
-            onChange={(event) => {
-                const value = event.target.value;
-                onChange(value === "__none__" ? null : value);
-            }}
-            placeholder={"Autogenerate ID"}
-            renderValue={(value) => {
-                return <Typography variant={"body2"}>
-                    {value !== "__none__" ? value : "Autogenerate ID"}
-                </Typography>;
-            }}
-            label={"Column that will be used as ID for each document"}>
-            <SelectItem value={"__none__"}>Autogenerate ID</SelectItem>
-            {Object.entries(headersMapping).map(([key, value]) => {
-                return <SelectItem key={key} value={key}>{key}</SelectItem>;
-            })}
-        </Select>
-    </div>;
-}
-
-function DefaultValuesField({
-    property,
-    onValueChange,
-    defaultValue
-}: { property: Property, onValueChange: (value: any) => void, defaultValue?: any }) {
-    const { t } = useTranslation();
-    const defaultValueLabel = t("default_value");
-    // Named fields stay distinguishable from each other — several of these sit
-    // in one table, and an aria-label repeated verbatim tells a screen reader
-    // nothing about which row it is on.
-    //
-    // Composed with a separator rather than a phrase: there is no key for the
-    // possessive form, and `t` returns the key itself for one that does not
-    // exist, so inventing `default_value_for` here would read out the literal
-    // string "default_value_for" in every locale, English included.
-    const ariaLabel = property.name
-        ? `${defaultValueLabel}: ${property.name}`
-        : defaultValueLabel;
-    if (property.type === "string") {
-        return <TextField size={"medium"}
-            aria-label={ariaLabel}
-            placeholder={defaultValueLabel}
-            value={defaultValue ?? ""}
-            onChange={(event) => onValueChange(event.target.value)}/>;
-    } else if (property.type === "number") {
-        return <TextField size={"medium"}
-            type={"number"}
-            aria-label={ariaLabel}
-            value={defaultValue ?? ""}
-            placeholder={defaultValueLabel}
-            onChange={(event) => onValueChange(event.target.value)}/>;
-    } else if (property.type === "boolean") {
-        return <BooleanSwitchWithLabel
-            value={defaultValue ?? null}
-            allowIndeterminate={true}
-            size={"small"}
-            onValueChange={(v: boolean | null) => onValueChange(v === null ? undefined : v)}
-            label={defaultValue === undefined
-                ? "Do not set value"
-                : defaultValue === true
-                    ? "Set value to true"
-                    : "Set value to false"}
-        />
-    } else if (property.type === "date") {
-        return <DateTimeField
-            mode={property.mode ?? "date"}
-            size={"medium"}
-            value={defaultValue ?? undefined}
-            onChange={(dateValue: Date | null) => {
-                onValueChange(dateValue);
-            }}
-            clearable={true}
-        />
-    }
-
-    return null;
-}