@firecms/data_import_export 3.0.0-canary.2 → 3.0.0-canary.200

package/src/export_import/ImportCollectionAction.tsx

@@ -1,420 +0,0 @@
-import React, { useCallback, useEffect } from "react";
-import {
-    CollectionActionsProps,
-    EntityCollectionTable,
-    PropertyConfigBadge,
-    getFieldConfig,
-    getPropertiesWithPropertiesOrder,
-    getPropertyInPath,
-    Property,
-    resolveCollection,
-    ResolvedProperties,
-    useCustomizationController,
-    User,
-    useSelectionController,
-    useSnackbarController
-} from "@firecms/core";
-import {
-    Button,
-    cn,
-    defaultBorderMixin,
-    Dialog,
-    DialogActions,
-    DialogContent,
-    FileUploadIcon,
-    IconButton,
-    Select,
-    SelectItem,
-    Tooltip,
-    Typography,
-} from "@firecms/ui";
-import { buildEntityPropertiesFromData } from "@firecms/schema_inference";
-import { useImportConfig } from "../hooks";
-import { convertDataToEntity, getInferenceType, getPropertiesMapping } from "../utils";
-import { DataNewPropertiesMapping, ImportFileUpload, ImportSaveInProgress } from "../components";
-import { ImportConfig } from "../types";
-
-type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
-
-export function ImportCollectionAction<M extends Record<string, any>, UserType extends User>({
-                                                                                                 collection,
-                                                                                                 path,
-                                                                                                 collectionEntitiesCount,
-                                                                                             }: CollectionActionsProps<M, UserType>
-) {
-    const customizationController = useCustomizationController();
-
-    const snackbarController = useSnackbarController();
-
-    const [open, setOpen] = React.useState(false);
-
-    const [step, setStep] = React.useState<ImportState>("initial");
-
-    const importConfig = useImportConfig();
-
-    const handleClickOpen = useCallback(() => {
-        setOpen(true);
-        setStep("initial");
-    }, [setOpen]);
-
-    const handleClose = useCallback(() => {
-        setOpen(false);
-    }, [setOpen]);
-
-    const onMappingComplete = useCallback(() => {
-        setStep("preview");
-    }, []);
-
-    const onPreviewComplete = useCallback(() => {
-        setStep("import_data_saving");
-    }, []);
-
-    const onDataAdded = async (data: object[]) => {
-        importConfig.setImportData(data);
-
-        if (data.length > 0) {
-            const originProperties = await buildEntityPropertiesFromData(data, getInferenceType);
-            importConfig.setOriginProperties(originProperties);
-
-            const headersMapping = buildHeadersMappingFromData(data);
-            importConfig.setHeadersMapping(headersMapping);
-            const firstKey = Object.keys(headersMapping)?.[0];
-            if (firstKey?.includes("id") || firstKey?.includes("key")) {
-                const idColumn = firstKey;
-                importConfig.setIdColumn(idColumn);
-            }
-        }
-        setTimeout(() => {
-            setStep("mapping");
-        }, 100);
-        // setStep("mapping");
-    };
-
-    const resolvedCollection = resolveCollection({
-        collection,
-        path,
-        fields: customizationController.propertyConfigs
-    });
-
-    const properties = getPropertiesWithPropertiesOrder<M>(resolvedCollection.properties, resolvedCollection.propertiesOrder as Extract<keyof M, string>[]) as ResolvedProperties<M>;
-
-    const propertiesAndLevel = Object.entries(properties)
-        .flatMap(([key, property]) => getPropertiesAndLevel(key, property, 0));
-    const propertiesOrder = (resolvedCollection.propertiesOrder ?? Object.keys(resolvedCollection.properties)) as Extract<keyof M, string>[];
-    if (collection.collectionGroup) {
-        return null;
-    }
-    return <>
-
-        <Tooltip title={"Import"}>
-            <IconButton color={"primary"} onClick={handleClickOpen}>
-                <FileUploadIcon/>
-            </IconButton>
-        </Tooltip>
-
-        <Dialog open={open}
-                fullWidth={step === "preview"}
-                fullHeight={step === "preview"}
-                maxWidth={step === "initial" ? "lg" : "7xl"}>
-            <DialogContent className={"flex flex-col gap-4 my-4"} fullHeight={step === "preview"}>
-
-                {step === "initial" && <>
-                    <Typography variant={"h6"}>Import data</Typography>
-                    <Typography variant={"body2"}>Upload a CSV, Excel or JSON file and map it to your existing
-                        schema</Typography>
-                    <ImportFileUpload onDataAdded={onDataAdded}/>
-                </>}
-
-                {step === "mapping" && <>
-                    <Typography variant={"h6"}>Map fields</Typography>
-                    <DataNewPropertiesMapping headersMapping={importConfig.headersMapping}
-                                              idColumn={importConfig.idColumn}
-                                              originProperties={importConfig.originProperties}
-                                              destinationProperties={properties}
-                                              onIdPropertyChanged={(value) => importConfig.setIdColumn(value)}
-                                              buildPropertyView={({
-                                                                      isIdColumn,
-                                                                      property,
-                                                                      propertyKey,
-                                                                      importKey
-                                                                  }) => {
-                                                  return <PropertyTreeSelect
-                                                      selectedPropertyKey={propertyKey ?? ""}
-                                                      properties={properties}
-                                                      propertiesAndLevel={propertiesAndLevel}
-                                                      isIdColumn={isIdColumn}
-                                                      onIdSelected={() => {
-                                                          importConfig.setIdColumn(importKey);
-                                                      }}
-                                                      onPropertySelected={(newPropertyKey) => {
-
-                                                          const newHeadersMapping: Record<string, string | null> = Object.entries(importConfig.headersMapping)
-                                                              .map(([currentImportKey, currentPropertyKey]) => {
-                                                                  if (currentPropertyKey === newPropertyKey) {
-                                                                      return { [currentImportKey]: null };
-                                                                  }
-                                                                  if (currentImportKey === importKey) {
-                                                                      return { [currentImportKey]: newPropertyKey };
-                                                                  }
-                                                                  return { [currentImportKey]: currentPropertyKey };
-                                                              })
-                                                              .reduce((acc, curr) => ({ ...acc, ...curr }), {});
-                                                          importConfig.setHeadersMapping(newHeadersMapping as Record<string, string>);
-
-                                                          if (newPropertyKey === importConfig.idColumn) {
-                                                              importConfig.setIdColumn(undefined);
-                                                          }
-
-                                                      }}
-                                                  />;
-                                              }}/>
-                </>}
-
-                {step === "preview" && <ImportDataPreview importConfig={importConfig}
-                                                          properties={properties}
-                                                          propertiesOrder={propertiesOrder}/>}
-
-                {step === "import_data_saving" && importConfig &&
-                    <ImportSaveInProgress importConfig={importConfig}
-                                          collection={collection}
-                                          onImportSuccess={(importedCollection) => {
-                                              handleClose();
-                                              snackbarController.open({
-                                                  type: "info",
-                                                  message: "Data imported successfully"
-                                              });
-                                          }}
-                    />}
-
-            </DialogContent>
-            <DialogActions>
-
-                {step === "mapping" && <Button onClick={() => setStep("initial")}
-                                               variant={"text"}>
-                    Back
-                </Button>}
-
-                {step === "preview" && <Button onClick={() => setStep("mapping")}
-                                               variant={"text"}>
-                    Back
-                </Button>}
-
-                <Button onClick={handleClose}
-                        variant={"text"}>
-                    Cancel
-                </Button>
-
-                {step === "mapping" && <Button variant="filled"
-                                               onClick={onMappingComplete}>
-                    Next
-                </Button>}
-
-                {step === "preview" && <Button variant="filled"
-                                               onClick={onPreviewComplete}>
-                    Save data
-                </Button>}
-
-            </DialogActions>
-        </Dialog>
-
-    </>;
-}
-
-const internalIDValue = "__internal_id__";
-
-function PropertyTreeSelect({
-                                selectedPropertyKey,
-                                properties,
-                                onPropertySelected,
-                                onIdSelected,
-                                propertiesAndLevel,
-                                isIdColumn
-                            }: {
-    selectedPropertyKey: string | null;
-    properties: Record<string, Property>;
-    onPropertySelected: (propertyKey: string | null) => void;
-    onIdSelected: () => void;
-    propertiesAndLevel: PropertyAndLevel[];
-    isIdColumn?: boolean;
-}) {
-
-    const selectedProperty = selectedPropertyKey ? getPropertyInPath(properties, selectedPropertyKey) : null;
-
-    const renderValue = useCallback((selectedPropertyKey: string) => {
-
-        if (selectedPropertyKey === internalIDValue) {
-            return <Typography variant={"body2"} className={"p-4"}>Use this column as ID</Typography>;
-        }
-
-        if (!selectedPropertyKey || !selectedProperty) {
-            return <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>;
-        }
-
-        return <PropertySelectEntry propertyKey={selectedPropertyKey}
-                                    property={selectedProperty as Property}/>;
-    }, [selectedProperty]);
-
-    const onSelectValueChange = useCallback((value: string) => {
-        if (value === internalIDValue) {
-            onIdSelected();
-            onPropertySelected(null);
-        } else if (value === "") {
-            onPropertySelected(null);
-        } else {
-            onPropertySelected(value);
-        }
-    }, []);
-
-    return <Select value={isIdColumn ? internalIDValue : (selectedPropertyKey ?? undefined)}
-                   onValueChange={onSelectValueChange}
-                   renderValue={renderValue}>
-
-        <SelectItem value={""}>
-            <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>
-        </SelectItem>
-
-        <SelectItem value={internalIDValue}>
-            <Typography variant={"body2"} className={"p-4"}>Use this column as ID</Typography>
-        </SelectItem>
-
-        {propertiesAndLevel.map(({
-                                     property,
-                                     level,
-                                     propertyKey
-                                 }) => {
-            return <SelectItem value={propertyKey}
-                               key={propertyKey}
-                               disabled={property.dataType === "map"}>
-                <PropertySelectEntry propertyKey={propertyKey}
-                                     property={property}
-                                     level={level}/>
-            </SelectItem>;
-        })}
-
-    </Select>;
-}
-
-type PropertyAndLevel = {
-    property: Property,
-    level: number,
-    propertyKey: string
-};
-
-function getPropertiesAndLevel(key: string, property: Property, level: number): PropertyAndLevel[] {
-    const properties: PropertyAndLevel[] = [];
-    properties.push({
-        property,
-        level,
-        propertyKey: key
-    });
-    if (property.dataType === "map" && property.properties) {
-        Object.entries(property.properties).forEach(([childKey, value]) => {
-            properties.push(...getPropertiesAndLevel(`${key}.${childKey}`, value as Property, level + 1));
-        });
-    }
-    return properties;
-}
-
-export function PropertySelectEntry({
-                                        propertyKey,
-                                        property,
-                                        level = 0
-                                    }: {
-    propertyKey: string;
-    property: Property;
-    level?: number;
-}) {
-
-    const { propertyConfigs } = useCustomizationController();
-    const widget = getFieldConfig(property, propertyConfigs);
-
-    return <div
-        className="flex flex-row w-full text-start items-center h-full">
-
-        {new Array(level).fill(0).map((_, index) =>
-            <div className={cn(defaultBorderMixin, "ml-8 border-l h-12")} key={index}/>)}
-
-        <div className={"m-4"}>
-            <Tooltip title={widget?.name}>
-                <PropertyConfigBadge propertyConfig={widget}/>
-            </Tooltip>
-        </div>
-
-        <div className={"flex flex-col flex-grow p-2 pl-2"}>
-            <Typography variant="body1"
-                        component="span"
-                        className="flex-grow pr-2">
-                {property.name
-                    ? property.name
-                    : "\u00a0"
-                }
-            </Typography>
-
-            <Typography className=" pr-2"
-                        variant={"body2"}
-                        component="span"
-                        color="secondary">
-                {propertyKey}
-            </Typography>
-        </div>
-
-    </div>;
-
-}
-
-export function ImportDataPreview<M extends Record<string, any>>({
-                                                                     importConfig,
-                                                                     properties,
-                                                                     propertiesOrder
-                                                                 }: {
-    importConfig: ImportConfig,
-    properties: ResolvedProperties<M>,
-    propertiesOrder: Extract<keyof M, string>[]
-}) {
-
-    useEffect(() => {
-        const propertiesMapping = getPropertiesMapping(importConfig.originProperties, properties);
-        const mappedData = importConfig.importData.map(d => convertDataToEntity(d, importConfig.idColumn, importConfig.headersMapping, properties, propertiesMapping, "TEMP_PATH"));
-        importConfig.setEntities(mappedData);
-    }, []);
-
-    const selectionController = useSelectionController();
-
-    return <EntityCollectionTable
-        title={<div>
-            <Typography variant={"subtitle2"}>Imported data preview</Typography>
-            <Typography variant={"caption"}>Entities with the same id will be overwritten</Typography>
-        </div>}
-        tableController={{
-            data: importConfig.entities,
-            dataLoading: false,
-            noMoreToLoad: false
-        }}
-        endAdornment={<div className={"h-12"}/>}
-        filterable={false}
-        sortable={false}
-        selectionController={selectionController}
-        displayedColumnIds={propertiesOrder.map(p => ({
-            key: p,
-            disabled: false
-        }))}
-        properties={properties}/>
-
-}
-
-function buildHeadersMappingFromData(objArr: object[]) {
-    const headersMapping: Record<string, string> = {};
-    objArr.filter(Boolean).forEach((obj) => {
-        Object.keys(obj).forEach((key) => {
-            // @ts-ignore
-            const child = obj[key];
-            if (typeof child === "object" && !Array.isArray(child)) {
-                Object.entries(buildHeadersMappingFromData([child])).forEach(([subKey, mapping]) => {
-                    headersMapping[`${key}.${subKey}`] = `${key}.${mapping}`;
-                });
-            }
-            headersMapping[key] = key;
-        });
-    });
-    return headersMapping;
-}

package/src/export_import/export.ts

@@ -1,200 +0,0 @@
-import {
-    ArrayValuesCount,
-    Entity,
-    EntityReference,
-    getArrayValuesCount,
-    getValueInPath,
-    ResolvedEntityCollection,
-    ResolvedProperties,
-    ResolvedProperty
-} from "@firecms/core";
-
-interface Header {
-    key: string;
-    label: string;
-}
-
-export function downloadExport<M extends Record<string, any>>(data: Entity<M>[],
-                                                              additionalData: Record<string, any>[] | undefined,
-                                                              collection: ResolvedEntityCollection<M>,
-                                                              flattenArrays: boolean,
-                                                              additionalHeaders: string[] | undefined,
-                                                              exportType: "csv" | "json",
-                                                              dateExportType: "timestamp" | "string"
-) {
-
-    console.debug("Downloading export", { dataLength: data.length, collection, exportType, dateExportType });
-    const properties = collection.properties;
-
-    if (exportType === "csv") {
-        const arrayValuesCount = flattenArrays ? getArrayValuesCount(data.map(d => d.values)) : {};
-        const headers = getExportHeaders(properties, additionalHeaders, arrayValuesCount);
-        const exportableData = getCSVExportableData(data, additionalData, properties, headers, dateExportType);
-        const headersData = entryToCSVRow(headers.map(h => h.label));
-        const csvData = exportableData.map(entry => entryToCSVRow(entry));
-        downloadBlob([headersData, ...csvData], `${collection.name}.csv`, "text/csv");
-    } else {
-        const exportableData = getJsonExportableData(data, additionalData, properties, dateExportType);
-        const json = JSON.stringify(exportableData, null, 2);
-        downloadBlob([json], `${collection.name}.json`, "application/json");
-    }
-}
-
-export function getCSVExportableData(data: Entity<any>[],
-                                     additionalData: Record<string, any>[] | undefined,
-                                     properties: ResolvedProperties,
-                                     headers: Header[],
-                                     dateExportType: "timestamp" | "string"
-) {
-
-    const mergedData: any[] = data.map(e => ({
-        id: e.id,
-        ...processValuesForExport(e.values, properties, "csv", dateExportType)
-    }));
-
-    if (additionalData) {
-        additionalData.forEach((additional, index) => {
-            mergedData[index] = { ...mergedData[index], ...additional };
-        });
-    }
-
-    return mergedData && mergedData.map((entry) => {
-        return headers.map((header) => getValueInPath(entry, header.key));
-    });
-}
-
-export function getJsonExportableData(data: Entity<any>[],
-                                      additionalData: Record<string, any>[] | undefined,
-                                      properties: ResolvedProperties,
-                                      dateExportType: "timestamp" | "string"
-) {
-
-    const mergedData: any[] = data.map(e => ({
-        id: e.id,
-        ...processValuesForExport(e.values, properties, "json", dateExportType)
-    }));
-
-    if (additionalData) {
-        additionalData.forEach((additional, index) => {
-            mergedData[index] = { ...mergedData[index], ...additional };
-        });
-    }
-
-    return mergedData;
-}
-
-function getExportHeaders<M extends Record<string, any>>(properties: ResolvedProperties<M>,
-                                                         additionalHeaders: string[] | undefined,
-                                                         arrayValuesCount?: ArrayValuesCount): Header[] {
-
-    const headers: Header[] = [
-        { label: "id", key: "id" },
-        ...Object.entries(properties)
-            .flatMap(([childKey, property]) => {
-                if (arrayValuesCount && arrayValuesCount[childKey] > 1) {
-                    return Array.from({ length: arrayValuesCount[childKey] },
-                        (_, i) => getHeaders(property as ResolvedProperty, `${childKey}[${i}]`, ""))
-                        .flat();
-                } else {
-                    return getHeaders(property as ResolvedProperty, childKey, "");
-                }
-            })
-    ];
-
-    if (additionalHeaders) {
-        headers.push(...additionalHeaders.map(h => ({ label: h, key: h })));
-    }
-
-    return headers;
-}
-
-/**
- * Get headers for property. There could be more than one header per property
- * @param property
- * @param propertyKey
- * @param prefix
- */
-function getHeaders(property: ResolvedProperty, propertyKey: string, prefix = ""): Header[] {
-    const currentKey = prefix ? `${prefix}.${propertyKey}` : propertyKey;
-    if (property.dataType === "map" && property.properties) {
-        return Object.entries(property.properties)
-            .map(([childKey, p]) => getHeaders(p, childKey, currentKey))
-            .flat();
-    } else {
-        return [{ label: currentKey, key: currentKey }];
-    }
-}
-
-function processValueForExport(inputValue: any,
-                               property: ResolvedProperty,
-                               exportType: "csv" | "json",
-                               dateExportType: "timestamp" | "string"
-): any {
-
-    let value;
-    if (property.dataType === "map" && property.properties) {
-        value = processValuesForExport(inputValue, property.properties as ResolvedProperties, exportType, dateExportType);
-    } else if (property.dataType === "array") {
-        if (property.of && Array.isArray(inputValue)) {
-            if (Array.isArray(property.of)) {
-                value = property.of.map((p, i) => processValueForExport(inputValue[i], p, exportType, dateExportType));
-            } else if (property.of.dataType === "map") {
-                value = exportType === "csv"
-                    ? inputValue.map((e) => JSON.stringify(e))
-                    : inputValue.map((e) => processValueForExport(e, property.of as ResolvedProperty, exportType, dateExportType));
-                ;
-            } else {
-                value = inputValue.map((e) => processValueForExport(e, property.of as ResolvedProperty, exportType, dateExportType));
-            }
-        } else {
-            value = inputValue;
-        }
-    } else if (property.dataType === "reference" && inputValue instanceof EntityReference) {
-        const ref = inputValue ? inputValue as EntityReference : undefined;
-        value = ref ? ref.pathWithId : null;
-    } else if (property.dataType === "date" && inputValue instanceof Date) {
-        value = inputValue ? (dateExportType === "timestamp" ? inputValue.getTime() : inputValue.toISOString()) : null;
-    } else {
-        value = inputValue;
-    }
-
-    return value;
-}
-
-function processValuesForExport<M extends Record<string, any>>
-(inputValues: Record<keyof M, any>,
- properties: ResolvedProperties<M>,
- exportType: "csv" | "json",
- dateExportType: "timestamp" | "string"
-): Record<keyof M, any> {
-    const updatedValues = Object.entries(properties)
-        .map(([key, property]) => {
-            const inputValue = inputValues && (inputValues)[key];
-            const updatedValue = processValueForExport(inputValue, property as ResolvedProperty, exportType, dateExportType);
-            if (updatedValue === undefined) return {};
-            return ({ [key]: updatedValue });
-        })
-        .reduce((a, b) => ({ ...a, ...b }), {}) as Record<keyof M, any>;
-    return { ...inputValues, ...updatedValues };
-}
-
-function entryToCSVRow(entry: any[]) {
-    return entry
-        .map((v: any) => {
-            if (v === null || v === undefined) return "";
-            if (Array.isArray(v))
-                return "\"" + JSON.stringify(v).replaceAll("\"", "\\\"") + "\"";
-            const s = String(v);
-            return "\"" + s.replaceAll("\"", "\"\"") + "\"";
-        })
-        .join(",") + "\r\n";
-}
-
-export function downloadBlob(content: BlobPart[], filename: string, contentType: string) {
-    const blob = new Blob(content, { type: contentType });
-    const url = URL.createObjectURL(blob);
-    const pom = document.createElement("a");
-    pom.href = url;
-    pom.setAttribute("download", filename);
-    pom.click();
-}

package/src/hooks/index.ts

@@ -1 +0,0 @@
-export * from "./useImportConfig";

package/src/hooks/useImportConfig.tsx

@@ -1,28 +0,0 @@
-import { useState } from "react";
-import { Entity, Property } from "@firecms/core";
-import { ImportConfig } from "../types";
-
-export const useImportConfig = (): ImportConfig => {
-
-    const [inUse, setInUse] = useState<boolean>(false);
-    const [idColumn, setIdColumn] = useState<string | undefined>();
-    const [importData, setImportData] = useState<object[]>([]);
-    const [entities, setEntities] = useState<Entity<any>[]>([]);
-    const [headersMapping, setHeadersMapping] = useState<Record<string, string | null>>({});
-    const [originProperties, setOriginProperties] = useState<Record<string, Property>>({});
-
-    return {
-        inUse,
-        setInUse,
-        idColumn,
-        setIdColumn,
-        entities,
-        setEntities,
-        importData,
-        setImportData,
-        headersMapping,
-        setHeadersMapping,
-        originProperties,
-        setOriginProperties,
-    };
-};

package/src/types/column_mapping.ts

@@ -1,32 +0,0 @@
-import React from "react";
-import { DataType, Entity, Property } from "@firecms/core";
-
-export type ImportConfig = {
-
-    inUse: boolean;
-    setInUse: React.Dispatch<React.SetStateAction<boolean>>;
-
-    idColumn: string | undefined;
-    setIdColumn: React.Dispatch<React.SetStateAction<string | undefined>>;
-
-    importData: object[];
-    setImportData: React.Dispatch<React.SetStateAction<object[]>>;
-
-    entities: Entity<any>[];
-    setEntities: React.Dispatch<React.SetStateAction<Entity<any>[]>>;
-
-    // mapping of the column name in the import file to the property key in the data model
-    headersMapping: Record<string, string | null>;
-    setHeadersMapping: React.Dispatch<React.SetStateAction<Record<string, string | null>>>;
-
-    originProperties: Record<string, Property>;
-    setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
-
-}
-
-export type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
-}

package/src/types/index.ts

@@ -1 +0,0 @@
-export * from "./column_mapping";