@rebasepro/cms 0.0.1-canary.4d4fb3e

package/src/data_import/import/ImportCollectionAction.tsx

@@ -0,0 +1,448 @@
+import React, { useCallback, useEffect } from "react";
+import {
+    useAuthController,
+    useCustomizationController,
+    useSnackbarController
+} from "@rebasepro/core";
+import { getPropertiesWithPropertiesOrder, getPropertyInPath } from "../../util";
+import { CollectionActionsProps, Properties, Property, User, EntityCollection } from "@rebasepro/types";
+import {
+    EntityCollectionTable,
+    getFieldConfig,
+    PropertyConfigBadge,
+    useSelectionController,
+} from "../../components";
+import { useCollectionRegistryController } from "../../hooks";
+import {
+    Button,
+    cls,
+    defaultBorderMixin,
+    Dialog,
+    DialogActions,
+    DialogContent,
+    DialogTitle,
+    IconButton,
+    Select,
+    SelectItem,
+    Tooltip,
+    Typography,
+    UploadIcon,
+} from "@rebasepro/ui";
+import { buildEntityPropertiesFromData } from "@rebasepro/schema-inference";
+import { useImportConfig } from "../hooks";
+import { convertDataToEntity, getInferenceType } from "../utils";
+import { DataNewPropertiesMapping, ImportFileUpload, ImportSaveInProgress } from "../components";
+import { ImportConfig } from "../types";
+import { slugify } from "@rebasepro/utils";
+
+type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
+
+export function ImportCollectionAction<M extends Record<string, unknown>, USER extends User>({
+    collection,
+    path,
+    onAnalyticsEvent
+}: CollectionActionsProps<M, USER> & {
+    onAnalyticsEvent?: (event: string, params?: any) => void;
+}
+) {
+
+    const snackbarController = useSnackbarController();
+
+    const [open, setOpen] = React.useState(false);
+
+    const [step, setStep] = React.useState<ImportState>("initial");
+
+    const importConfig = useImportConfig();
+
+    const handleClickOpen = useCallback(() => {
+        setOpen(true);
+        onAnalyticsEvent?.("import_open");
+        setStep("initial");
+    }, [onAnalyticsEvent]);
+
+    const handleClose = useCallback(() => {
+        setOpen(false);
+    }, [setOpen]);
+
+    const onMappingComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_mapping_complete");
+        setStep("preview");
+    }, [onAnalyticsEvent]);
+
+    const onPreviewComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_data_save");
+        setStep("import_data_saving");
+    }, [onAnalyticsEvent]);
+
+    const onDataAdded = async (data: object[]) => {
+        importConfig.setImportData(data);
+
+        if (data.length > 0) {
+            const originProperties = await buildEntityPropertiesFromData(data, getInferenceType);
+            importConfig.setOriginProperties(originProperties as any);
+
+            const headersMapping = buildHeadersMappingFromData(data, collection?.properties);
+            importConfig.setHeadersMapping(headersMapping);
+            const firstKey = Object.keys(headersMapping)?.[0];
+            if (firstKey?.includes("id") || firstKey?.includes("key")) {
+                importConfig.setIdColumn(firstKey);
+            }
+        }
+        setTimeout(() => {
+            onAnalyticsEvent?.("import_data_added");
+            setStep("mapping");
+        }, 100);
+    };
+
+    const properties = getPropertiesWithPropertiesOrder(collection.properties, collection.propertiesOrder as Extract<keyof M, string>[]);
+
+    const propertiesAndLevel = Object.entries(properties)
+        .flatMap(([key, property]) => getPropertiesAndLevel(key, property, 0));
+    const propertiesOrder = (collection.propertiesOrder ?? Object.keys(collection.properties)) as Extract<keyof M, string>[];
+
+    return <>
+
+        <Tooltip title={"Import"}
+            asChild={true}>
+            <IconButton
+                size={"small"}
+                color={"primary"} onClick={handleClickOpen}>
+                <UploadIcon size={"small"} />
+            </IconButton>
+        </Tooltip>
+
+        <Dialog open={open}
+            fullWidth={step !== "initial"}
+            fullHeight={step !== "initial"}
+            maxWidth={step === "initial" ? "lg" : "7xl"}>
+
+            <DialogTitle variant={"h6"} hidden={step === "preview"}>Import data</DialogTitle>
+
+            <DialogContent className={"h-full flex flex-col gap-4 my-4"} fullHeight={step === "preview"}>
+
+                {step === "initial" && <>
+                    <Typography variant={"body2"}>Upload a CSV, Excel or JSON file and map it to your existing
+                        schema</Typography>
+                    <ImportFileUpload onDataAdded={onDataAdded} />
+                </>}
+
+                {step === "mapping" && <>
+                    <DataNewPropertiesMapping importConfig={importConfig}
+                        destinationProperties={properties}
+                        buildPropertyView={({
+                            isIdColumn,
+                            property,
+                            propertyKey,
+                            importKey,
+                        }) => {
+                            return <PropertyTreeSelect
+                                selectedPropertyKey={propertyKey ?? ""}
+                                properties={properties}
+                                propertiesAndLevel={propertiesAndLevel}
+                                isIdColumn={isIdColumn}
+                                onIdSelected={() => {
+                                    importConfig.setIdColumn(importKey);
+                                }}
+                                onPropertySelected={(newPropertyKey) => {
+
+                                    onAnalyticsEvent?.("import_mapping_field_updated");
+                                    const newHeadersMapping: Record<string, string | null> = Object.entries(importConfig.headersMapping)
+                                        .map(([currentImportKey, currentPropertyKey]) => {
+                                            if (currentPropertyKey === newPropertyKey) {
+                                                return { [currentImportKey]: null };
+                                            }
+                                            if (currentImportKey === importKey) {
+                                                return { [currentImportKey]: newPropertyKey };
+                                            }
+                                            return { [currentImportKey]: currentPropertyKey };
+                                        })
+                                        .reduce((acc, curr) => ({ ...acc, ...curr }), {});
+                                    importConfig.setHeadersMapping(newHeadersMapping as Record<string, string>);
+
+                                    if (newPropertyKey === importConfig.idColumn) {
+                                        importConfig.setIdColumn(undefined);
+                                    }
+
+                                }}
+                            />;
+                        }} />
+                </>}
+
+                {step === "preview" && <ImportDataPreview importConfig={importConfig}
+                    properties={properties}
+                    propertiesOrder={propertiesOrder} />}
+
+                {step === "import_data_saving" && importConfig &&
+                    <ImportSaveInProgress importConfig={importConfig}
+                        collection={collection as unknown as EntityCollection}
+                        path={path}
+                        onImportSuccess={(importedCollection) => {
+                            handleClose();
+                            snackbarController.open({
+                                type: "info",
+                                message: "Data imported successfully"
+                            });
+                        }}
+                    />}
+
+            </DialogContent>
+            <DialogActions>
+
+                {step === "mapping" && <Button
+                    onClick={() => setStep("initial")}
+                    variant={"text"}>
+                    Back
+                </Button>}
+
+                {step === "preview" && <Button
+                    onClick={() => setStep("mapping")}
+                    variant={"text"}>
+                    Back
+                </Button>}
+
+                <Button onClick={handleClose}
+                    variant={"text"}>
+                    Cancel
+                </Button>
+
+                {step === "mapping" && <Button variant="filled"
+                    color={"primary"}
+                    onClick={onMappingComplete}>
+                    Next
+                </Button>}
+
+                {step === "preview" && <Button variant="filled"
+                    color={"primary"}
+                    onClick={onPreviewComplete}>
+                    Save data
+                </Button>}
+
+            </DialogActions>
+        </Dialog>
+
+    </>;
+}
+
+const internalIDValue = "__internal_id__";
+
+function PropertyTreeSelect({
+    selectedPropertyKey,
+    properties,
+    onPropertySelected,
+    onIdSelected,
+    propertiesAndLevel,
+    isIdColumn
+}: {
+    selectedPropertyKey: string | null;
+    properties: Record<string, Property>;
+    onPropertySelected: (propertyKey: string | null) => void;
+    onIdSelected: () => void;
+    propertiesAndLevel: PropertyAndLevel[];
+    isIdColumn?: boolean;
+}) {
+
+    const selectedProperty = selectedPropertyKey ? getPropertyInPath(properties, selectedPropertyKey) : null;
+
+    const renderValue = useCallback((selectedPropertyKey: string) => {
+
+        if (selectedPropertyKey === internalIDValue) {
+            return <Typography variant={"body2"} className={"p-4"}>Use this column as ID</Typography>;
+        }
+
+        if (!selectedPropertyKey || !selectedProperty) {
+            return <Typography variant={"body2"} color="disabled" className={"p-4"}>Do not import this
+                property</Typography>;
+        }
+
+        return <PropertySelectEntry propertyKey={selectedPropertyKey}
+            property={selectedProperty as Property} />;
+    }, [selectedProperty]);
+
+    const onSelectValueChange = (value: string) => {
+        if (value === internalIDValue) {
+            onIdSelected();
+            onPropertySelected(null);
+        } else if (value === "__do_not_import") {
+            onPropertySelected(null);
+        } else {
+            onPropertySelected(value);
+        }
+    };
+
+    return <Select value={isIdColumn ? internalIDValue : (selectedPropertyKey ?? undefined)}
+        fullWidth={true}
+        onValueChange={onSelectValueChange}
+        renderValue={renderValue}>
+
+        <SelectItem value={"__do_not_import"}>
+            <Typography variant={"body2"} color={"disabled"} className={"p-4"}>Do not import this property</Typography>
+        </SelectItem>
+
+        <SelectItem value={internalIDValue}>
+            <Typography variant={"body2"} className={"p-4"}>Use this column as ID</Typography>
+        </SelectItem>
+
+        {propertiesAndLevel.map(({
+            property,
+            level,
+            propertyKey
+        }) => {
+            return <SelectItem value={propertyKey}
+                key={propertyKey}
+                disabled={property.type === "map"}>
+                <PropertySelectEntry propertyKey={propertyKey}
+                    property={property}
+                    level={level} />
+            </SelectItem>;
+        })}
+
+    </Select>;
+}
+
+type PropertyAndLevel = {
+    property: Property,
+    level: number,
+    propertyKey: string
+};
+
+function getPropertiesAndLevel(key: string, property: Property, level: number): PropertyAndLevel[] {
+    const properties: PropertyAndLevel[] = [];
+    properties.push({
+        property,
+        level,
+        propertyKey: key
+    });
+    if (property.type === "map" && property.properties) {
+        Object.entries(property.properties).forEach(([childKey, value]) => {
+            properties.push(...getPropertiesAndLevel(`${key}.${childKey}`, value as Property, level + 1));
+        });
+    }
+    return properties;
+}
+
+export function PropertySelectEntry({
+    propertyKey,
+    property,
+    level = 0
+}: {
+    propertyKey: string;
+    property: Property;
+    level?: number;
+}) {
+
+    const { propertyConfigs } = useCustomizationController();
+    const widget = getFieldConfig(property, propertyConfigs);
+
+    return <div
+        className="flex flex-row w-full text-start items-center h-full">
+
+        {new Array(level).fill(0).map((_, index) =>
+            <div className={cls(defaultBorderMixin, "ml-8 border-l h-12")} key={index} />)}
+
+        <div className={"m-4"}>
+            <Tooltip title={widget?.name}>
+                <PropertyConfigBadge propertyConfig={widget} />
+            </Tooltip>
+        </div>
+
+        <div className={"flex flex-col grow p-2 pl-2"}>
+            <Typography variant="body1"
+                component="span"
+                className="grow pr-2">
+                {property.name
+                    ? property.name
+                    : "\u00a0"
+                }
+            </Typography>
+
+            <Typography className=" pr-2"
+                variant={"body2"}
+                component="span"
+                color="secondary">
+                {propertyKey}
+            </Typography>
+        </div>
+
+    </div>;
+
+}
+
+export function ImportDataPreview<M extends Record<string, unknown>>({
+    importConfig,
+    properties,
+    propertiesOrder
+}: {
+    importConfig: ImportConfig,
+    properties: Properties,
+    propertiesOrder: Extract<keyof M, string>[],
+}) {
+    const authController = useAuthController();
+    const collectionRegistry = useCollectionRegistryController();
+    useEffect(() => {
+        const mappedData = importConfig.importData.map(d => convertDataToEntity(
+            authController,
+            collectionRegistry,
+            d,
+            importConfig.idColumn,
+            importConfig.headersMapping,
+            properties,
+            "TEMP_PATH",
+            importConfig.defaultValues
+        ));
+        importConfig.setEntities(mappedData);
+    }, []);
+
+    const selectionController = useSelectionController();
+
+    return <EntityCollectionTable
+        title={<div>
+            <Typography variant={"subtitle2"}>Imported data preview</Typography>
+            <Typography variant={"caption"}>Entities with the same id will be overwritten</Typography>
+        </div>}
+        tableController={{
+            data: importConfig.entities,
+            dataLoading: false,
+            noMoreToLoad: false
+        }}
+        enablePopupIcon={false}
+        endAdornment={<div className={"h-12"} />}
+        filterable={false}
+        sortable={false}
+        openEntityMode={"full_screen"}
+        selectionController={selectionController}
+        properties={properties} />
+
+}
+
+function buildHeadersMappingFromData(objArr: object[], properties?: Properties) {
+    const headersMapping: Record<string, string> = {};
+    objArr.filter(Boolean).forEach((obj) => {
+        Object.keys(obj).forEach((key) => {
+            // @ts-ignore
+            const child = obj[key];
+            if (typeof child === "object" && !Array.isArray(child)) {
+                const childProperty = properties?.[key];
+                const childProperties = childProperty && "properties" in childProperty ? childProperty.properties : undefined;
+                const childHeadersMapping = buildHeadersMappingFromData([child], childProperties);
+                Object.entries(childHeadersMapping).forEach(([subKey, mapping]) => {
+                    headersMapping[`${key}.${subKey}`] = `${key}.${mapping}`;
+                });
+            }
+
+            if (!properties) {
+                headersMapping[key] = key;
+            } else if (key in properties) {
+                headersMapping[key] = key;
+            } else {
+                const slug = slugify(key);
+                if (slug in properties) {
+                    headersMapping[key] = slug;
+                } else {
+                    headersMapping[key] = key;
+                }
+            }
+
+        });
+    });
+    return headersMapping;
+}

package/src/data_import/import/index.ts

@@ -0,0 +1 @@
+export * from "./ImportCollectionAction";

package/src/data_import/index.ts

@@ -0,0 +1,5 @@
+export * from "./components";
+export * from "./types";
+export * from "./utils";
+export * from "./hooks";
+export * from "./import";

package/src/data_import/types/column_mapping.ts

@@ -0,0 +1,32 @@
+import React from "react";
+import { DataType, Entity, Property } from "@rebasepro/types";
+
+export type ImportConfig = {
+
+    inUse: boolean;
+    setInUse: React.Dispatch<React.SetStateAction<boolean>>;
+
+    idColumn: string | undefined;
+    setIdColumn: React.Dispatch<React.SetStateAction<string | undefined>>;
+
+    importData: object[];
+    setImportData: React.Dispatch<React.SetStateAction<object[]>>;
+
+    entities: Entity<any>[];
+    setEntities: React.Dispatch<React.SetStateAction<Entity<any>[]>>;
+
+    // mapping of the column name in the import file to the property key in the data model
+    headersMapping: Record<string, string | null>;
+    setHeadersMapping: React.Dispatch<React.SetStateAction<Record<string, string | null>>>;
+
+    originProperties: Record<string, Property>;
+    setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
+
+    // unmapped headings order
+    headingsOrder: string[];
+    setHeadingsOrder: React.Dispatch<React.SetStateAction<string[]>>;
+
+    defaultValues: Record<string, any>;
+    setDefaultValues: React.Dispatch<React.SetStateAction<Record<string, any>>>;
+
+}

package/src/data_import/types/index.ts

@@ -0,0 +1 @@
+export * from "./column_mapping";

package/src/data_import/utils/data.ts

@@ -0,0 +1,145 @@
+import { getPropertyInPath } from "../../util";
+import { AuthController, Entity, EntityCollection, EntityReference, CollectionRegistryController, Properties, Property } from "@rebasepro/types";
+import { isPropertyBuilder } from "@rebasepro/common";
+import { unflattenObject } from "./file_to_json";
+import { getIn } from "@rebasepro/formex";
+import { inferTypeFromValue } from "@rebasepro/schema-inference";
+import { mergeDeep } from "@rebasepro/utils";
+
+export function convertDataToEntity(authController: AuthController,
+    navigation: CollectionRegistryController,
+    data: Record<any, any>,
+    idColumn: string | undefined,
+    headersMapping: Record<string, string | null>,
+    properties: Properties,
+    path: string,
+    defaultValues: Record<string, any>): Entity<any> {
+    const flatObject = flattenEntry(data);
+    if (idColumn)
+        delete flatObject[idColumn];
+    const mappedKeysObject = Object.entries(flatObject)
+        .map(([key, value]) => {
+            const mappedKey = (getIn(headersMapping, key) as string | undefined) ?? key;
+
+            const mappedProperty = getPropertyInPath(properties, mappedKey);
+            if (!mappedProperty) {
+                return {};
+            }
+            const processedValue = processValueMapping(authController, value, navigation, mappedProperty);
+            return ({
+                [mappedKey]: processedValue
+            });
+        })
+        .reduce((acc, curr) => ({ ...acc, ...curr }), {});
+
+    const values = mergeDeep(defaultValues ?? {}, unflattenObject(mappedKeysObject));
+    let id = idColumn ? data[idColumn] : undefined;
+    if (typeof id === "string") {
+        id = id.trim();
+    } else if (typeof id === "number") {
+        id = id.toString();
+    } else if (typeof id === "boolean") {
+        id = id.toString();
+    } else if (id instanceof Date) {
+        id = id.toISOString();
+    } else if (id && "toString" in id) {
+        id = id.toString();
+    }
+
+    return {
+        id,
+        values,
+        path: path
+    };
+}
+
+export function flattenEntry(obj: any, parent = ""): any {
+    return Object.keys(obj).reduce((acc, key) => {
+        const prefixedKey = parent ? `${parent}.${key}` : key;
+
+        if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
+            Object.assign(acc, flattenEntry(obj[key], prefixedKey));
+        } else {
+            // @ts-ignore
+            acc[prefixedKey] = obj[key];
+        }
+
+        return acc;
+    }, {});
+}
+
+export function processValueMapping(authController: AuthController, value: any, navigation: CollectionRegistryController, property?: Property): any {
+    if (value === null) return null;
+
+    if (property === undefined) return value;
+    const usedProperty: Property | null = property;
+    if (usedProperty === null) return value;
+    const from = inferTypeFromValue(value);
+    const to = usedProperty.type;
+
+    if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !Array.isArray(usedProperty.of) && !isPropertyBuilder(usedProperty.of)) {
+        return value.map(v => processValueMapping(authController, v, navigation, usedProperty.of as Property));
+    } else if (from === "string" && to === "number" && typeof value === "string") {
+        return Number(value);
+    } else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !Array.isArray(usedProperty.of) && !isPropertyBuilder(usedProperty.of)) {
+        return value.split(",").map((v: string) => processValueMapping(authController, v, navigation, usedProperty.of as Property));
+    } else if (from === "string" && to === "boolean") {
+        return value === "true";
+    } else if (from === "number" && to === "boolean") {
+        return value === 1;
+    } else if (from === "boolean" && to === "number") {
+        return value ? 1 : 0;
+    } else if (from === "boolean" && to === "string") {
+        return value ? "true" : "false";
+    } else if (from === "number" && to === "string" && typeof value === "number") {
+        return value.toString();
+    } else if (from === "string" && to === "array" && typeof value === "string") {
+        return value.split(",").map((v: string) => v.trim());
+    } else if (from === "string" && to === "date" && typeof value === "string") {
+        try {
+            return new Date(value);
+        } catch (e) {
+            return value;
+        }
+    } else if (value instanceof Date && to === "string") {
+        return value.toISOString();
+    } else if (from === "number" && to === "date" && typeof value === "number") {
+        try {
+            return new Date(value);
+        } catch (e) {
+            return value;
+        }
+    } else if (from === "string" && to === "reference" && typeof value === "string") {
+        // Parse optional database name in format: database_name:::path/to/entity
+        // or simple format: path/to/entity
+        let databaseId: string | undefined = undefined;
+        let referencePath = value;
+
+        if (value.includes(":::")) {
+            const [dbName, pathPart] = value.split(":::");
+            if (dbName && dbName !== "(default)") {
+                databaseId = dbName;
+            }
+            referencePath = pathPart;
+        }
+
+        // split value into path and entityId (entityId is the last part of the path, after the last /)
+        const path = referencePath.split("/").slice(0, -1).join("/");
+        const entityId = referencePath.split("/").slice(-1)[0];
+
+        // If no explicit database was provided in the string, try to get it from the collection
+        if (databaseId === undefined) {
+            const targetCollection: EntityCollection<any> | undefined = navigation.getCollection(path);
+            databaseId = targetCollection?.databaseId;
+        }
+
+        return new EntityReference({ id: entityId, path, databaseId });
+
+    } else if (from === to) {
+        return value;
+    } else if (from === "array" && to === "string" && Array.isArray(value)) {
+        return value.join(",");
+    }
+
+    return value;
+}