@firecms/data_import_export 3.0.0-canary.5 → 3.0.0-canary.51

package/dist/utils/data.d.ts

@@ -1,11 +1,4 @@
-import { DataType, Entity, Properties } from "@firecms/core";
-type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
-};
-export declare function convertDataToEntity(data: Record<any, any>, idColumn: string | undefined, headersMapping: Record<string, string | null>, properties: Properties, propertiesMapping: Record<string, DataTypeMapping>, path: string): Entity<any>;
+import { Entity, Properties, PropertyOrBuilder } from "@firecms/core";
+export declare function convertDataToEntity(data: Record<any, any>, idColumn: string | undefined, headersMapping: Record<string, string | null>, properties: Properties, path: string, defaultValues: Record<string, any>): Entity<any>;
 export declare function flattenEntry(obj: any, parent?: string): any;
-export declare function processValueMapping(value: any, valueMapping?: DataTypeMapping): any;
-export {};
+export declare function processValueMapping(value: any, property?: PropertyOrBuilder): any;

package/dist/utils/file_headers.d.ts

@@ -0,0 +1 @@
+export declare function getXLSXHeaders(sheet: any): any[];

package/dist/utils/file_to_json.d.ts

@@ -1,4 +1,8 @@
-export declare function convertFileToJson(file: File): Promise<object[]>;
+type ConversionResult = {
+    data: object[];
+    propertiesOrder: string[];
+};
+export declare function convertFileToJson(file: File): Promise<ConversionResult>;
 /**
  * Take an object with keys of type `address.street`, `address.city` and
  * convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
@@ -9,3 +13,4 @@ export declare function unflattenObject(flatObj: {
 }): {
     [key: string]: any;
 };
+export {};

package/dist/utils/get_properties_mapping.d.ts

@@ -1,3 +0,0 @@
-import { Properties } from "@firecms/core";
-import { DataTypeMapping } from "../types";
-export declare function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping>;

package/dist/utils/index.d.ts

@@ -1,4 +1,3 @@
 export * from "./file_to_json";
 export * from "./data";
 export * from "./get_import_inference_type";
-export * from "./get_properties_mapping";

package/package.json

@@ -1,29 +1,31 @@
 {
   "name": "@firecms/data_import_export",
   "type": "module",
-  "version": "3.0.0-canary.5",
+  "version": "3.0.0-canary.51",
   "access": "public",
   "main": "./dist/index.umd.js",
   "module": "./dist/index.es.js",
   "types": "./dist/index.d.ts",
   "source": "src/index.ts",
   "dependencies": {
-    "@firecms/core": "^3.0.0-canary.5",
-    "@firecms/schema_inference": "^3.0.0-canary.5",
+    "@firecms/core": "^3.0.0-canary.51",
+    "@firecms/formex": "^3.0.0-canary.51",
+    "@firecms/schema_inference": "^3.0.0-canary.51",
+    "@firecms/ui": "^3.0.0-canary.51",
     "xlsx": "^0.18.5"
   },
   "peerDependencies": {
     "firebase": "^10.7.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "react-router": "^6.12.0",
-    "react-router-dom": "^6.12.0"
+    "react-router": "^6.22.0",
+    "react-router-dom": "^6.22.0"
   },
   "exports": {
     ".": {
       "import": "./dist/index.es.js",
       "require": "./dist/index.umd.js",
-      "types": "./dist/src/index.d.ts"
+      "types": "./dist/index.d.ts"
     },
     "./package.json": "./package.json"
   },
@@ -54,25 +56,25 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
-    "@testing-library/jest-dom": "^5.17.0",
+    "@testing-library/jest-dom": "^6.4.5",
     "@types/jest": "^29.5.12",
-    "@types/react": "^18.2.54",
-    "@types/react-dom": "^18.2.18",
-    "@typescript-eslint/eslint-plugin": "^5.62.0",
-    "@typescript-eslint/parser": "^5.62.0",
+    "@types/react": "^18.3.1",
+    "@types/react-dom": "^18.3.0",
+    "@typescript-eslint/eslint-plugin": "^7.8.0",
+    "@typescript-eslint/parser": "^7.8.0",
     "@vitejs/plugin-react": "^4.2.1",
     "babel-jest": "^29.7.0",
-    "eslint": "^8.56.0",
+    "eslint": "^8.57.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-n": "^15.7.0",
+    "eslint-plugin-n": "^16.6.2",
     "eslint-plugin-promise": "^6.1.1",
-    "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react": "^7.34.1",
+    "eslint-plugin-react-hooks": "^4.6.2",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.2",
-    "typescript": "^5.3.3",
-    "vite": "^5.1.1",
+    "typescript": "^5.4.5",
+    "vite": "^5.2.11",
     "vite-plugin-fonts": "^0.7.0"
   },
   "jest": {
@@ -99,5 +101,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "e3850e04c28ef87d561bdf651950c7fbac87a9ec"
+  "gitHead": "d4d30347803b07a09f5b003eb63c78e46eb055fa"
 }

package/src/components/DataNewPropertiesMapping.tsx

@@ -1,6 +1,9 @@
-import { getPropertyInPath, Property, } from "@firecms/core";
+import { getPropertyInPath, PropertiesOrBuilders, Property } from "@firecms/core";
 import {
+    BooleanSwitchWithLabel,
     ChevronRightIcon,
+    DateTimeField,
+    ExpandablePanel,
     Select,
     SelectItem,
     Table,
@@ -8,15 +11,15 @@ import {
     TableCell,
     TableHeader,
     TableRow,
+    TextField,
     Typography
 } from "@firecms/ui";
+import { ImportConfig } from "../types";
+import { getIn, setIn } from "@firecms/formex";
 
 export interface DataPropertyMappingProps {
-    idColumn?: string;
-    headersMapping: Record<string, string | null>;
-    originProperties: Record<string, Property>;
+    importConfig: ImportConfig;
     destinationProperties: Record<string, Property>;
-    onIdPropertyChanged: (value: string) => void;
     buildPropertyView?: (props: {
         isIdColumn: boolean,
         property: Property | null,
@@ -26,20 +29,24 @@ export interface DataPropertyMappingProps {
 }
 
 export function DataNewPropertiesMapping({
-                                             idColumn,
-                                             headersMapping,
-                                             originProperties,
+                                             importConfig,
                                              destinationProperties,
-                                             onIdPropertyChanged,
-                                             buildPropertyView,
+                                             buildPropertyView
                                          }: DataPropertyMappingProps) {
 
+    const headersMapping = importConfig.headersMapping;
+    const headingsOrder = importConfig.headingsOrder;
+    const idColumn = importConfig.idColumn;
+    const originProperties = importConfig.originProperties;
+
     return (
         <>
 
             <IdSelectField idColumn={idColumn}
                            headersMapping={headersMapping}
-                           onChange={onIdPropertyChanged}/>
+                           onChange={(value) => importConfig.setIdColumn(value ?? undefined)}/>
+
+            <div className={"h-4"}/>
 
             <Table style={{
                 tableLayout: "fixed"
@@ -51,51 +58,113 @@ export function DataNewPropertiesMapping({
                     <TableCell header={true}>
                     </TableCell>
                     <TableCell header={true} style={{ width: "75%" }}>
-                        Property
+                        Map to Property
                     </TableCell>
                 </TableHeader>
                 <TableBody>
                     {destinationProperties &&
-                        Object.entries(headersMapping)
-                            .map(([importKey, mappedKey]) => {
-                                    const propertyKey = headersMapping[importKey];
-                                    const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
+                        headingsOrder.map((importKey) => {
+                                const mappedKey = headersMapping[importKey];
+                                const propertyKey = headersMapping[importKey];
+                                const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
 
-                                    const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
-                                    const originDataType = originProperty ? (originProperty.dataType === "array" && typeof originProperty.of === "object"
-                                            ? `${originProperty.dataType} - ${(originProperty.of as Property).dataType}`
-                                            : originProperty.dataType)
-                                        : undefined;
-                                    return <TableRow key={importKey} style={{ height: "90px" }}>
+                                const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
+                                const originDataType = originProperty
+                                    ? (originProperty.dataType === "array" && typeof originProperty.of === "object"
+                                        ? `${originProperty.dataType} - ${(originProperty.of as Property).dataType}`
+                                        : originProperty.dataType)
+                                    : undefined;
+                                return <TableRow key={importKey} style={{ height: "90px" }}>
+                                    <TableCell style={{ width: "20%" }}>
+                                        <Typography variant={"body2"}>{importKey}</Typography>
+                                        {originProperty && <Typography
+                                            variant={"caption"}
+                                            color={"secondary"}
+                                        >{originDataType}</Typography>}
+                                    </TableCell>
+                                    <TableCell>
+                                        <ChevronRightIcon/>
+                                    </TableCell>
+                                    <TableCell className={importKey === idColumn ? "text-center" : undefined}
+                                               style={{ width: "75%" }}>
+                                        {buildPropertyView?.({
+                                            isIdColumn: importKey === idColumn,
+                                            property,
+                                            propertyKey,
+                                            importKey
+                                        })}
+                                    </TableCell>
+                                </TableRow>;
+                            }
+                        )}
+                </TableBody>
+            </Table>
+
+            <ExpandablePanel title="Default values" initiallyExpanded={false} className={"p-4 mt-4"}>
+
+                <div className={"text-sm text-slate-500 dark:text-slate-300 font-medium ml-3.5 mb-1"}>
+                    You can select a default value for unmapped columns and empty values:
+                </div>
+                <Table style={{
+                    tableLayout: "fixed"
+                }}>
+                    <TableHeader>
+                        <TableCell header={true} style={{ width: "30%" }}>
+                            Property
+                        </TableCell>
+                        <TableCell header={true}>
+                        </TableCell>
+                        <TableCell header={true} style={{ width: "65%" }}>
+                            Default value
+                        </TableCell>
+                    </TableHeader>
+                    <TableBody>
+                        {destinationProperties &&
+                            getAllPropertyKeys(destinationProperties).map((key) => {
+                                    const property = getPropertyInPath(destinationProperties, key);
+                                    if (typeof property !== "object" || property === null) {
+                                        return null;
+                                    }
+                                    if (!["number", "string", "boolean", "map"].includes(property.dataType)) {
+                                        return null;
+                                    }
+                                    return <TableRow key={key} style={{ height: "70px" }}>
                                         <TableCell style={{ width: "20%" }}>
-                                            <Typography variant={"body2"}>{importKey}</Typography>
-                                            {originProperty && <Typography
-                                                variant={"caption"}
-                                                color={"secondary"}
-                                            >{originDataType}</Typography>}
+                                            <Typography variant={"body2"}>{key}</Typography>
                                         </TableCell>
                                         <TableCell>
                                             <ChevronRightIcon/>
                                         </TableCell>
-                                        <TableCell className={importKey === idColumn ? "text-center" : undefined}
+                                        <TableCell className={key === idColumn ? "text-center" : undefined}
                                                    style={{ width: "75%" }}>
-                                            {buildPropertyView?.({
-                                                isIdColumn: importKey === idColumn,
-                                                property,
-                                                propertyKey,
-                                                importKey
-                                            })
-                                            }
+                                            <DefaultValuesField property={property}
+                                                                defaultValue={getIn(importConfig.defaultValues, key)}
+                                                                onValueChange={(value) => {
+                                                                    const newValues = setIn(importConfig.defaultValues, key, value);
+                                                                    importConfig.setDefaultValues(newValues);
+                                                                }}/>
                                         </TableCell>
                                     </TableRow>;
                                 }
                             )}
-                </TableBody>
-            </Table>
+                    </TableBody>
+                </Table>
+            </ExpandablePanel>
         </>
     );
 }
 
+function getAllPropertyKeys(properties: PropertiesOrBuilders, currentKey?: string): string[] {
+    return Object.entries(properties).reduce((acc, [key, property]) => {
+        const accumulatedKey = currentKey ? `${currentKey}.${key}` : key;
+        if (typeof property !== "function" && property.dataType === "map" && property.properties) {
+            const childProperties = getAllPropertyKeys(property.properties, accumulatedKey);
+            return [...acc, ...childProperties];
+        }
+        return [...acc, accumulatedKey];
+    }, [] as string[]);
+}
+
 function IdSelectField({
                            idColumn,
                            headersMapping,
@@ -103,25 +172,70 @@ function IdSelectField({
                        }: {
     idColumn?: string,
     headersMapping: Record<string, string | null>;
-    onChange: (value: string) => void
+    onChange: (value: string | null) => void
 }) {
     return <div>
         <Select
             size={"small"}
             value={idColumn ?? ""}
             onChange={(event) => {
-                onChange(event.target.value as string);
+                const value = event.target.value;
+                onChange(value === "__none__" ? null : value);
             }}
+            placeholder={"Autogenerate ID"}
             renderValue={(value) => {
                 return <Typography variant={"body2"}>
-                    {value !== "" ? value : "Autogenerate ID"}
+                    {value !== "__none__" ? value : "Autogenerate ID"}
                 </Typography>;
             }}
             label={"Column that will be used as ID for each document"}>
-            <SelectItem value={""}>Autogenerate ID</SelectItem>
+            <SelectItem value={"__none__"}>Autogenerate ID</SelectItem>
             {Object.entries(headersMapping).map(([key, value]) => {
                 return <SelectItem key={key} value={key}>{key}</SelectItem>;
             })}
         </Select>
     </div>;
 }
+
+function DefaultValuesField({
+                                property,
+                                onValueChange,
+                                defaultValue
+                            }: { property: Property, onValueChange: (value: any) => void, defaultValue?: any }) {
+    if (property.dataType === "string") {
+        return <TextField size={"small"}
+                          placeholder={"Default value"}
+                          value={defaultValue ?? ""}
+                          onChange={(event) => onValueChange(event.target.value)}/>;
+    } else if (property.dataType === "number") {
+        return <TextField size={"small"}
+                          type={"number"}
+                          value={defaultValue ?? ""}
+                          placeholder={"Default value"}
+                          onChange={(event) => onValueChange(event.target.value)}/>;
+    } else if (property.dataType === "boolean") {
+        return <BooleanSwitchWithLabel
+            value={defaultValue ?? null}
+            allowIndeterminate={true}
+            size={"small"}
+            onValueChange={(v: boolean | null) => onValueChange(v === null ? undefined : v)}
+            label={defaultValue === undefined
+                ? "Do not set value"
+                : defaultValue === true
+                    ? "Set value to true"
+                    : "Set value to false"}
+        />
+    } else if (property.dataType === "date") {
+        return <DateTimeField
+            mode={property.mode ?? "date"}
+            size={"small"}
+            value={defaultValue ?? undefined}
+            onChange={(dateValue: Date | undefined) => {
+                onValueChange(dateValue);
+            }}
+            clearable={true}
+        />
+    }
+
+    return null;
+}

package/src/components/ImportFileUpload.tsx

@@ -2,7 +2,9 @@ import { FileUpload, UploadIcon } from "@firecms/ui";
 import { convertFileToJson } from "../utils/file_to_json";
 import { useSnackbarController } from "@firecms/core";
 
-export function ImportFileUpload({ onDataAdded }: { onDataAdded: (data: object[]) => void }) {
+export function ImportFileUpload({ onDataAdded }: {
+    onDataAdded: (data: object[], propertiesOrder?: string[]) => void
+}) {
     const snackbarController = useSnackbarController();
     return <FileUpload
         accept={{
@@ -22,12 +24,18 @@ export function ImportFileUpload({ onDataAdded }: { onDataAdded: (data: object[]
         onFilesAdded={(files: File[]) => {
             if (files.length > 0) {
                 convertFileToJson(files[0])
-                    .then((jsonData) => {
-                        onDataAdded(jsonData);
+                    .then(({
+                               data,
+                               propertiesOrder
+                           }) => {
+                        onDataAdded(data, propertiesOrder);
                     })
                     .catch((error) => {
                         console.error("Error parsing file", error);
-                        snackbarController.open({ type: "error", message: error.message });
+                        snackbarController.open({
+                            type: "error",
+                            message: error.message
+                        });
                     });
             }
         }}/>

package/src/components/ImportNewPropertyFieldPreview.tsx

@@ -1,5 +1,11 @@
 import React from "react";
-import { ErrorBoundary, PropertyConfigBadge, getFieldConfig, Property, useCustomizationController } from "@firecms/core";
+import {
+    ErrorBoundary,
+    getFieldConfig,
+    Property,
+    PropertyConfigBadge,
+    useCustomizationController
+} from "@firecms/core";
 import { EditIcon, IconButton, TextField, } from "@firecms/ui";
 
 export function ImportNewPropertyFieldPreview({
@@ -49,7 +55,6 @@ export function ImportNewPropertyFieldPreview({
 
             </div>
 
-
         </div>
     </ErrorBoundary>
 }

package/src/components/ImportSaveInProgress.tsx

@@ -1,5 +1,5 @@
 import { DataSource, Entity, EntityCollection, useDataSource } from "@firecms/core";
-import { CenteredView, CircularProgress, Typography, } from "@firecms/ui";
+import { Button, CenteredView, CircularProgress, Typography, } from "@firecms/ui";
 import { useEffect, useRef, useState } from "react";
 import { ImportConfig } from "../types";
 
@@ -17,7 +17,7 @@ export function ImportSaveInProgress<C extends EntityCollection>
          onImportSuccess: (collection: C) => void
      }) {
 
-    console.log("ImportSaveInProgress", path)
+    const [errorSaving, setErrorSaving] = useState<Error | undefined>(undefined);
     const dataSource = useDataSource();
 
     const savingRef = useRef<boolean>(false);
@@ -42,6 +42,9 @@ export function ImportSaveInProgress<C extends EntityCollection>
         ).then(() => {
             onImportSuccess(collection);
             savingRef.current = false;
+        }).catch((e) => {
+            setErrorSaving(e);
+            savingRef.current = false;
         });
     }
 
@@ -49,6 +52,25 @@ export function ImportSaveInProgress<C extends EntityCollection>
         save();
     }, []);
 
+    if (errorSaving) {
+        return (
+            <CenteredView className={"flex flex-col gap-4 items-center"}>
+                <Typography variant={"h6"}>
+                    Error saving data
+                </Typography>
+
+                <Typography variant={"body2"} color={"error"}>
+                    {errorSaving.message}
+                </Typography>
+                <Button
+                    onClick={save}
+                    variant={"outlined"}>
+                    Retry
+                </Button>
+            </CenteredView>
+        );
+    }
+
     return (
         <CenteredView className={"flex flex-col gap-4 items-center"}>
             <CircularProgress/>

package/src/export_import/BasicExportAction.tsx

@@ -0,0 +1,137 @@
+import React, { useCallback } from "react";
+
+import { Entity, ResolvedProperties } from "@firecms/core";
+import {
+    BooleanSwitchWithLabel,
+    Button,
+    cn,
+    Dialog,
+    DialogActions,
+    DialogContent,
+    focusedMixin,
+    GetAppIcon,
+    IconButton,
+    Tooltip,
+    Typography,
+} from "@firecms/ui";
+import { downloadEntitiesExport } from "./export";
+
+export type BasicExportActionProps = {
+    data: Entity<any>[];
+    properties: ResolvedProperties;
+    propertiesOrder?: string[];
+}
+
+export function BasicExportAction({
+                                      data,
+                                      properties,
+                                      propertiesOrder
+                                  }: BasicExportActionProps) {
+
+    const dateRef = React.useRef<Date>(new Date());
+    const [flattenArrays, setFlattenArrays] = React.useState<boolean>(true);
+    const [exportType, setExportType] = React.useState<"csv" | "json">("csv");
+    const [dateExportType, setDateExportType] = React.useState<"timestamp" | "string">("string");
+
+    const [open, setOpen] = React.useState(false);
+
+    const handleClickOpen = useCallback(() => {
+        setOpen(true);
+    }, [setOpen]);
+
+    const handleClose = useCallback(() => {
+        setOpen(false);
+    }, [setOpen]);
+
+    const onOkClicked = useCallback(() => {
+        downloadEntitiesExport(data, [], properties, propertiesOrder, "export.csv", flattenArrays, [], exportType, dateExportType);
+        handleClose();
+    }, []);
+
+    return <>
+
+        <Tooltip title={"Export"}>
+            <IconButton color={"primary"} onClick={handleClickOpen}>
+                <GetAppIcon/>
+            </IconButton>
+        </Tooltip>
+
+        <Dialog
+            open={open}
+            onOpenChange={setOpen}
+            maxWidth={"xl"}>
+            <DialogContent className={"flex flex-col gap-4 my-4"}>
+
+                <Typography variant={"h6"}>Export data</Typography>
+
+                <div>Download the the content of this table as a CSV</div>
+
+                <div className={"flex flex-row gap-4"}>
+                    <div className={"p-4 flex flex-col"}>
+                        <div className="flex items-center">
+                            <input id="radio-csv" type="radio" value="csv" name="exportType"
+                                   checked={exportType === "csv"}
+                                   onChange={() => setExportType("csv")}
+                                   className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
+                            <label htmlFor="radio-csv"
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">CSV</label>
+                        </div>
+                        <div className="flex items-center">
+                            <input id="radio-json" type="radio" value="json" name="exportType"
+                                   checked={exportType === "json"}
+                                   onChange={() => setExportType("json")}
+                                   className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
+                            <label htmlFor="radio-json"
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">JSON</label>
+                        </div>
+                    </div>
+
+                    <div className={"p-4 flex flex-col"}>
+                        <div className="flex items-center">
+                            <input id="radio-timestamp" type="radio" value="timestamp" name="dateExportType"
+                                   checked={dateExportType === "timestamp"}
+                                   onChange={() => setDateExportType("timestamp")}
+                                   className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
+                            <label htmlFor="radio-timestamp"
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">Dates as
+                                timestamps ({dateRef.current.getTime()})</label>
+                        </div>
+                        <div className="flex items-center">
+                            <input id="radio-string" type="radio" value="string" name="dateExportType"
+                                   checked={dateExportType === "string"}
+                                   onChange={() => setDateExportType("string")}
+                                   className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
+                            <label htmlFor="radio-string"
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">Dates as
+                                strings ({dateRef.current.toISOString()})</label>
+                        </div>
+                    </div>
+                </div>
+
+                <BooleanSwitchWithLabel
+                    size={"small"}
+                    disabled={exportType !== "csv"}
+                    value={flattenArrays}
+                    onValueChange={setFlattenArrays}
+                    label={"Flatten arrays"}/>
+
+            </DialogContent>
+
+            <DialogActions>
+
+                <Button onClick={handleClose}
+                        variant={"text"}>
+                    Cancel
+                </Button>
+
+                <Button variant="filled"
+                        onClick={onOkClicked}>
+                    Download
+                </Button>
+
+            </DialogActions>
+
+        </Dialog>
+
+    </>;
+}