@rebasepro/admin 0.0.1-canary.4f204c2

package/src/data_export/export/ExportCollectionAction.tsx

@@ -0,0 +1,266 @@
+
+import React, { useCallback } from "react";
+
+import {
+    useAuthController,
+    useCustomizationController,
+    useData
+} from "@rebasepro/core";
+import { useCMSContext } from "../../hooks";
+import { CollectionActionsProps, Entity, EntityCollection, ExportConfig, RebaseContext, User } from "@rebasepro/types";
+import { getDefaultValuesFor } from "@rebasepro/common";
+import { Alert, BooleanSwitchWithLabel, Button, CircularProgress, cls, Dialog, DialogActions, DialogContent, DialogTitle, IconButton, Tooltip , iconSize } from "@rebasepro/ui";
+import { DownloadIcon } from "lucide-react";
+import { downloadEntitiesExport } from "./export";
+
+const DOCS_LIMIT = 500;
+
+export function ExportCollectionAction<M extends Record<string, unknown>, USER extends User>({
+    collection,
+    path,
+    collectionEntitiesCount,
+    onAnalyticsEvent,
+    exportAllowed,
+    notAllowedView
+}: CollectionActionsProps<M, USER, EntityCollection<M>> & {
+    exportAllowed?: (props: { collectionEntitiesCount: number, path: string, collection: EntityCollection }) => boolean;
+    notAllowedView?: React.ReactNode;
+    onAnalyticsEvent?: (event: string, params?: any) => void;
+}) {
+
+    const exportConfig = typeof collection.exportable === "object" ? collection.exportable : undefined;
+
+    const dateRef = React.useRef<Date>(new Date());
+
+    const [includeUndefinedValues, setIncludeUndefinedValues] = React.useState<boolean>(false);
+    const [flattenArrays, setFlattenArrays] = React.useState<boolean>(true);
+    const [exportType, setExportType] = React.useState<"csv" | "json">("csv");
+    const [dateExportType, setDateExportType] = React.useState<"timestamp" | "string">("string");
+
+    const context = useCMSContext<USER>();
+    const dataClient = useData();
+
+    const canExport = !exportAllowed || exportAllowed({
+        collectionEntitiesCount: collectionEntitiesCount ?? 0,
+        path,
+        collection
+    });
+
+    const [dataLoading, setDataLoading] = React.useState<boolean>(false);
+    const [dataLoadingError, setDataLoadingError] = React.useState<Error | undefined>();
+
+    const [open, setOpen] = React.useState(false);
+
+    const handleClickOpen = useCallback(() => {
+        setOpen(true);
+    }, [setOpen]);
+
+    const handleClose = useCallback(() => {
+        setOpen(false);
+    }, [setOpen]);
+
+    const fetchAdditionalFields = useCallback(async (entities: Entity<M>[]) => {
+
+        const additionalExportFields = exportConfig?.additionalFields;
+        const additionalFields = collection.additionalFields;
+
+        const resolvedExportColumnsValues: Record<string, any>[] = additionalExportFields
+            ? await Promise.all(entities.map(async (entity) => {
+                return (await Promise.all(additionalExportFields.map(async (column) => {
+                    return {
+                        [column.key]: await column.builder({
+                            entity,
+                            context: context as unknown as RebaseContext
+                        })
+                    };
+                }))).reduce((a, b) => ({ ...a,
+...b }), {});
+            }))
+            : [];
+
+        const resolvedColumnsValues: Record<string, any>[] = additionalFields
+            ? await Promise.all(entities.map(async (entity) => {
+                return (await Promise.all(additionalFields
+                    .map(async (field) => {
+                        if (!field.value)
+                            return {};
+                        return {
+                            [field.key]: await field.value({
+                                entity,
+                                context: context as unknown as RebaseContext
+                            })
+                        };
+                    }))).reduce((a, b) => ({ ...a,
+...b }), {});
+            }))
+            : [];
+        return [...resolvedExportColumnsValues, ...resolvedColumnsValues];
+    }, [exportConfig?.additionalFields]);
+
+    const doDownload = useCallback(async (collection: EntityCollection<M>,
+        exportConfig: ExportConfig<any> | undefined) => {
+
+        onAnalyticsEvent?.("export_collection", {
+            collection: collection.slug
+        });
+        setDataLoading(true);
+        dataClient.collection(path).find({})
+            .then(async (res) => {
+                const data = res.data as Entity<M>[];
+                setDataLoadingError(undefined);
+                const additionalData = await fetchAdditionalFields(data);
+                const additionalHeaders = [
+                    ...exportConfig?.additionalFields?.map(column => column.key) ?? [],
+                    ...collection.additionalFields?.map(field => field.key) ?? []
+                ];
+
+                const dataWithDefaults = includeUndefinedValues
+                    ? data.map(entity => {
+                        const defaultValues = getDefaultValuesFor(collection.properties);
+                        return {
+                            ...entity,
+                            values: { ...defaultValues,
+...entity.values }
+                        };
+                    })
+                    : data;
+                downloadEntitiesExport({
+                    data: dataWithDefaults,
+                    additionalData,
+                    properties: collection.properties,
+                    propertiesOrder: collection.propertiesOrder,
+                    name: collection.name,
+                    flattenArrays,
+                    additionalHeaders,
+                    exportType,
+                    dateExportType
+                });
+                onAnalyticsEvent?.("export_collection_success", {
+                    collection: collection.slug
+                });
+            })
+            .catch((e) => {
+                console.error("Error loading export data", e);
+                setDataLoadingError(e);
+            })
+            .finally(() => setDataLoading(false));
+
+    }, [onAnalyticsEvent, dataClient, path, fetchAdditionalFields, includeUndefinedValues, flattenArrays, exportType, dateExportType]);
+
+    const onOkClicked = useCallback(() => {
+        doDownload(collection, exportConfig);
+        handleClose();
+    }, [doDownload, collection, exportConfig, handleClose]);
+
+    return <>
+
+        <Tooltip title={"Export"}
+            asChild={true}>
+            <IconButton
+                size={"small"}
+                color={"primary"}
+                onClick={handleClickOpen}>
+                <DownloadIcon size={iconSize.small}/>
+            </IconButton>
+        </Tooltip>
+
+        <Dialog
+            open={open}
+            onOpenChange={setOpen}
+            maxWidth={"xl"}>
+
+            <DialogTitle variant={"h6"}>Export data</DialogTitle>
+
+            <DialogContent className={"flex flex-col gap-4 my-4"}>
+
+                <div>DownloadIcon the the content of this table as a CSV</div>
+
+                {collectionEntitiesCount !== undefined && collectionEntitiesCount > DOCS_LIMIT &&
+                    <Alert color={"warning"}>
+                        <div>
+                            This collections has a large number
+                            of documents ({collectionEntitiesCount}).
+                        </div>
+                    </Alert>}
+
+                <div className={"flex flex-row gap-4"}>
+                    <div className={"p-4 flex flex-col"}>
+                        <div className="flex items-center">
+                            <input id="radio-csv" type="radio" value="csv" name="exportType"
+                                checked={exportType === "csv"}
+                                onChange={() => setExportType("csv")}
+                                className={cls("w-4 bg-surface-100 border-surface-300 dark:bg-surface-700 dark:border-surface-600")}/>
+                            <label htmlFor="radio-csv"
+                                className="p-2 text-sm font-medium text-surface-900 dark:text-surface-300">CSV</label>
+                        </div>
+                        <div className="flex items-center">
+                            <input id="radio-json" type="radio" value="json" name="exportType"
+                                checked={exportType === "json"}
+                                onChange={() => setExportType("json")}
+                                className={cls("w-4 bg-surface-100 border-surface-300 dark:bg-surface-700 dark:border-surface-600")}/>
+                            <label htmlFor="radio-json"
+                                className="p-2 text-sm font-medium text-surface-900 dark:text-surface-300">JSON</label>
+                        </div>
+                    </div>
+
+                    <div className={"p-4 flex flex-col"}>
+                        <div className="flex items-center">
+                            <input id="radio-timestamp" type="radio" value="timestamp" name="dateExportType"
+                                checked={dateExportType === "timestamp"}
+                                onChange={() => setDateExportType("timestamp")}
+                                className={cls("w-4 bg-surface-100 border-surface-300 dark:bg-surface-700 dark:border-surface-600")}/>
+                            <label htmlFor="radio-timestamp"
+                                className="p-2 text-sm font-medium text-surface-900 dark:text-surface-300">Dates
+                                as
+                                timestamps ({dateRef.current.getTime()})</label>
+                        </div>
+                        <div className="flex items-center">
+                            <input id="radio-string" type="radio" value="string" name="dateExportType"
+                                checked={dateExportType === "string"}
+                                onChange={() => setDateExportType("string")}
+                                className={cls("w-4 bg-surface-100 border-surface-300 dark:bg-surface-700 dark:border-surface-600")}/>
+                            <label htmlFor="radio-string"
+                                className="p-2 text-sm font-medium text-surface-900 dark:text-surface-300">Dates
+                                as
+                                strings ({dateRef.current.toISOString()})</label>
+                        </div>
+                    </div>
+                </div>
+
+                <BooleanSwitchWithLabel
+                    size={"small"}
+                    disabled={exportType !== "csv"}
+                    value={flattenArrays}
+                    onValueChange={setFlattenArrays}
+                    label={"Flatten arrays"}/>
+
+                <BooleanSwitchWithLabel
+                    size={"small"}
+                    value={includeUndefinedValues}
+                    onValueChange={setIncludeUndefinedValues}
+                    label={"Include undefined values"}/>
+
+                {!canExport && notAllowedView}
+
+            </DialogContent>
+
+            <DialogActions>
+
+                {dataLoading && <CircularProgress size={"smallest"}/>}
+
+                <Button onClick={handleClose}
+                    variant={"text"}>
+                    Cancel
+                </Button>
+
+                <Button onClick={onOkClicked}
+                    disabled={dataLoading || !canExport}>
+                    DownloadIcon
+                </Button>
+
+            </DialogActions>
+
+        </Dialog>
+
+    </>;
+}

package/src/data_export/export/export.ts

@@ -0,0 +1,248 @@
+import { Entity, EntityReference, Properties, Property } from "@rebasepro/types";
+import { type ArrayValuesCount, getArrayValuesCount, getValueInPath } from "@rebasepro/utils";
+
+interface Header {
+    key: string;
+    label: string;
+}
+
+export interface DownloadEntitiesExportParams<M extends Record<string, unknown>> {
+    data: Entity<M>[];
+    additionalData: Record<string, any>[] | undefined;
+    properties: Properties;
+    propertiesOrder: string[] | undefined;
+    name: string;
+    flattenArrays: boolean;
+    additionalHeaders: string[] | undefined;
+    exportType: "csv" | "json";
+    dateExportType: "timestamp" | "string";
+}
+
+export function downloadEntitiesExport<M extends Record<string, unknown>>({
+    data,
+    additionalData,
+    properties,
+    propertiesOrder,
+    name,
+    flattenArrays,
+    additionalHeaders,
+    exportType,
+    dateExportType
+}: DownloadEntitiesExportParams<M>
+) {
+
+    console.debug("Downloading export", {
+        dataLength: data.length,
+        properties,
+        exportType,
+        dateExportType
+    });
+
+    if (exportType === "csv") {
+        const arrayValuesCount = flattenArrays ? getArrayValuesCount(data.map(d => d.values)) : {};
+        const headers = getExportHeaders(properties, propertiesOrder, additionalHeaders, arrayValuesCount);
+        const exportableData = getEntityCSVExportableData(data, additionalData, properties, headers, dateExportType);
+        const headersData = entryToCSVRow(headers.map(h => h.label));
+        const csvData = exportableData.map(entry => entryToCSVRow(entry));
+        downloadBlob([headersData, ...csvData], `${name}.csv`, "text/csv");
+    } else {
+        const exportableData = getEntityJsonExportableData(data, additionalData, properties, dateExportType);
+        const json = JSON.stringify(exportableData, null, 2);
+        downloadBlob([json], `${name}.json`, "application/json");
+    }
+}
+
+export function getEntityCSVExportableData(data: Entity<any>[],
+    additionalData: Record<string, any>[] | undefined,
+    properties: Properties,
+    headers: Header[],
+    dateExportType: "timestamp" | "string"
+) {
+
+    const mergedData: any[] = data.map(e => ({
+        id: e.id,
+        ...processValuesForExport(e.values, properties, "csv", dateExportType)
+    }));
+
+    if (additionalData) {
+        additionalData.forEach((additional, index) => {
+            mergedData[index] = { ...mergedData[index],
+...additional };
+        });
+    }
+
+    return mergedData && mergedData.map((entry) => {
+        return headers.map((header) => getValueInPath(entry, header.key));
+    });
+}
+
+export function getEntityJsonExportableData(data: Entity<any>[],
+    additionalData: Record<string, any>[] | undefined,
+    properties: Properties,
+    dateExportType: "timestamp" | "string"
+) {
+
+    const mergedData: any[] = data.map(e => ({
+        id: e.id,
+        ...processValuesForExport(e.values, properties, "json", dateExportType)
+    }));
+
+    if (additionalData) {
+        additionalData.forEach((additional, index) => {
+            mergedData[index] = { ...mergedData[index],
+...additional };
+        });
+    }
+
+    return mergedData;
+}
+
+function getExportHeaders<M extends Record<string, unknown>>(properties: Properties,
+    propertiesOrder: string[] | undefined,
+    additionalHeaders: string[] | undefined,
+    arrayValuesCount?: ArrayValuesCount): Header[] {
+
+    const headers: Header[] = [
+        {
+            label: "id",
+            key: "id"
+        },
+        ...(propertiesOrder ?? Object.keys(properties))
+            .flatMap((childKey) => {
+                const property = properties[childKey];
+                if (!property) {
+                    console.warn("Property not found", childKey, properties);
+                    return [];
+                }
+                if (arrayValuesCount && arrayValuesCount[childKey] > 1) {
+                    return Array.from({ length: arrayValuesCount[childKey] },
+                        (_, i) => getHeaders(property as Property, `${childKey}[${i}]`, ""))
+                        .flat();
+                } else {
+                    return getHeaders(property as Property, childKey, "");
+                }
+            })
+    ];
+
+    if (additionalHeaders) {
+        headers.push(...additionalHeaders.map(h => ({
+            label: h,
+            key: h
+        })));
+    }
+
+    return headers;
+}
+
+/**
+ * Get headers for property. There could be more than one header per property
+ * @param property
+ * @param propertyKey
+ * @param prefix
+ */
+function getHeaders(property: Property, propertyKey: string, prefix = ""): Header[] {
+    const currentKey = prefix ? `${prefix}.${propertyKey}` : propertyKey;
+    if (property.type === "map" && property.properties) {
+        return Object.entries(property.properties)
+            .map(([childKey, p]) => getHeaders(p, childKey, currentKey))
+            .flat();
+    } else {
+        return [{
+            label: currentKey,
+            key: currentKey
+        }];
+    }
+}
+
+function processValueForExport(inputValue: any,
+    property: Property,
+    exportType: "csv" | "json",
+    dateExportType: "timestamp" | "string"
+): any {
+
+    let value;
+    if (property.type === "map" && property.properties) {
+        value = processValuesForExport(inputValue, property.properties as Properties, exportType, dateExportType);
+    } else if (property.type === "array") {
+        if (property.of && Array.isArray(inputValue)) {
+            if (Array.isArray(property.of)) {
+                value = property.of.map((p, i) => processValueForExport(inputValue[i], p, exportType, dateExportType));
+            } else if (property.of.type === "map") {
+                value = exportType === "csv"
+                    ? inputValue.map((e) => JSON.stringify(e))
+                    : inputValue.map((e) => processValueForExport(e, property.of as Property, exportType, dateExportType));
+                ;
+            } else {
+                value = inputValue.map((e) => processValueForExport(e, property.of as Property, exportType, dateExportType));
+            }
+        } else {
+            value = inputValue;
+        }
+    } else if (property.type === "reference" && inputValue && inputValue.isEntityReference && inputValue.isEntityReference()) {
+        const ref = inputValue ? inputValue as EntityReference : undefined;
+        value = ref ? ref.fullPath : null;
+    } else if (property.type === "date" && inputValue instanceof Date) {
+        value = inputValue ? (dateExportType === "timestamp" ? inputValue.getTime() : inputValue.toISOString()) : null;
+    } else {
+        value = inputValue;
+    }
+
+    return value;
+}
+
+function processValuesForExport<M extends Record<string, unknown>>
+    (inputValues: Record<keyof M, any>,
+        properties: Properties,
+        exportType: "csv" | "json",
+        dateExportType: "timestamp" | "string"
+    ): Record<keyof M, any> {
+    const updatedValues = Object.entries(properties)
+        .map(([key, property]) => {
+            const inputValue = inputValues && (inputValues)[key];
+            const updatedValue = processValueForExport(inputValue, property as Property, exportType, dateExportType);
+            if (updatedValue === undefined) return {};
+            return ({ [key]: updatedValue });
+        })
+        .reduce((a, b) => ({ ...a,
+...b }), {}) as Record<keyof M, any>;
+    return { ...inputValues,
+...updatedValues };
+}
+
+function entryToCSVRow(entry: any[]) {
+    return entry
+        .map((v: any) => {
+            if (v === null || v === undefined) return "";
+            if (Array.isArray(v))
+                return "\"" + JSON.stringify(v).replaceAll("\"", "\"\"") + "\"";
+            const s = String(v);
+            return "\"" + s.replaceAll("\"", "\"\"") + "\"";
+        })
+        .join(",") + "\r\n";
+}
+
+export function downloadBlob(content: BlobPart[], filename: string, contentType: string) {
+    const blob = new Blob(content, { type: contentType });
+    const url = URL.createObjectURL(blob);
+    const pom = document.createElement("a");
+    pom.href = url;
+    pom.setAttribute("download", filename);
+    pom.click();
+}
+
+export function downloadDataAsCsv(data: object[], name: string) {
+    if (data.length === 0) return;
+
+    const headers = Object.keys(data[0]);
+    const csvContent = [
+        headers.join(","),
+        ...data.map(row => headers.map(header => {
+            const value = (row as Record<string, unknown>)[header];
+            if (value === null || value === undefined) return "";
+            if (Array.isArray(value)) return `"${JSON.stringify(value).replace(/"/g, '""')}"`;
+            return `"${String(value).replace(/"/g, '""')}"`;
+        }).join(","))
+    ].join("\r\n");
+
+    downloadBlob([csvContent], `${name}.csv`, "text/csv");
+}

package/src/data_export/export/index.ts

@@ -0,0 +1,3 @@
+export * from "./export";
+export * from "./BasicExportAction";
+export * from "./ExportCollectionAction";

package/src/data_export/index.ts

@@ -0,0 +1 @@
+export * from "./export";