@firecms/data_import_export 3.0.0-canary.41 → 3.0.0-canary.43

package/dist/utils/get_properties_mapping.d.ts

@@ -1,3 +0,0 @@
-import { Properties } from "@firecms/core";
-import { DataTypeMapping } from "../types";
-export declare function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping>;

package/dist/utils/index.d.ts

@@ -1,4 +1,3 @@
 export * from "./file_to_json";
 export * from "./data";
 export * from "./get_import_inference_type";
-export * from "./get_properties_mapping";

package/package.json

@@ -1,15 +1,17 @@
 {
   "name": "@firecms/data_import_export",
   "type": "module",
-  "version": "3.0.0-canary.41",
+  "version": "3.0.0-canary.43",
   "access": "public",
   "main": "./dist/index.umd.js",
   "module": "./dist/index.es.js",
   "types": "./dist/index.d.ts",
   "source": "src/index.ts",
   "dependencies": {
-    "@firecms/core": "^3.0.0-canary.41",
-    "@firecms/schema_inference": "^3.0.0-canary.41",
+    "@firecms/core": "^3.0.0-canary.43",
+    "@firecms/formex": "^3.0.0-canary.43",
+    "@firecms/schema_inference": "^3.0.0-canary.43",
+    "@firecms/ui": "^3.0.0-canary.43",
     "xlsx": "^0.18.5"
   },
   "peerDependencies": {
@@ -56,10 +58,10 @@
     "@jest/globals": "^29.7.0",
     "@testing-library/jest-dom": "^6.4.2",
     "@types/jest": "^29.5.12",
-    "@types/react": "^18.2.67",
-    "@types/react-dom": "^18.2.22",
-    "@typescript-eslint/eslint-plugin": "^7.3.1",
-    "@typescript-eslint/parser": "^7.3.1",
+    "@types/react": "^18.2.79",
+    "@types/react-dom": "^18.2.25",
+    "@typescript-eslint/eslint-plugin": "^7.7.0",
+    "@typescript-eslint/parser": "^7.7.0",
     "@vitejs/plugin-react": "^4.2.1",
     "babel-jest": "^29.7.0",
     "eslint": "^8.57.0",
@@ -71,8 +73,8 @@
     "eslint-plugin-react-hooks": "^4.6.0",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.2",
-    "typescript": "^5.4.2",
-    "vite": "^5.2.3",
+    "typescript": "^5.4.5",
+    "vite": "^5.2.9",
     "vite-plugin-fonts": "^0.7.0"
   },
   "jest": {
@@ -99,5 +101,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "d1ffa185f8930bab4e7b9941ba551c53f947aa1f"
+  "gitHead": "fedcb0d43c504245dd76b702e4ab4479fa8592df"
 }

package/src/components/DataNewPropertiesMapping.tsx

@@ -1,6 +1,9 @@
-import { getPropertyInPath, Property, } from "@firecms/core";
+import { getPropertyInPath, PropertiesOrBuilders, Property } from "@firecms/core";
 import {
+    BooleanSwitchWithLabel,
     ChevronRightIcon,
+    DateTimeField,
+    ExpandablePanel,
     Select,
     SelectItem,
     Table,
@@ -8,15 +11,15 @@ import {
     TableCell,
     TableHeader,
     TableRow,
+    TextField,
     Typography
 } from "@firecms/ui";
+import { ImportConfig } from "../types";
+import { getIn, setIn } from "@firecms/formex";
 
 export interface DataPropertyMappingProps {
-    idColumn?: string;
-    headersMapping: Record<string, string | null>;
-    originProperties: Record<string, Property>;
+    importConfig: ImportConfig;
     destinationProperties: Record<string, Property>;
-    onIdPropertyChanged: (value: string | null) => void;
     buildPropertyView?: (props: {
         isIdColumn: boolean,
         property: Property | null,
@@ -26,20 +29,24 @@ export interface DataPropertyMappingProps {
 }
 
 export function DataNewPropertiesMapping({
-                                             idColumn,
-                                             headersMapping,
-                                             originProperties,
+                                             importConfig,
                                              destinationProperties,
-                                             onIdPropertyChanged,
-                                             buildPropertyView,
+                                             buildPropertyView
                                          }: DataPropertyMappingProps) {
 
+    const headersMapping = importConfig.headersMapping;
+    const headingsOrder = importConfig.headingsOrder;
+    const idColumn = importConfig.idColumn;
+    const originProperties = importConfig.originProperties;
+
     return (
         <>
 
             <IdSelectField idColumn={idColumn}
                            headersMapping={headersMapping}
-                           onChange={onIdPropertyChanged}/>
+                           onChange={(value) => importConfig.setIdColumn(value ?? undefined)}/>
+
+            <div className={"h-4"}/>
 
             <Table style={{
                 tableLayout: "fixed"
@@ -51,51 +58,113 @@ export function DataNewPropertiesMapping({
                     <TableCell header={true}>
                     </TableCell>
                     <TableCell header={true} style={{ width: "75%" }}>
-                        Property
+                        Map to Property
                     </TableCell>
                 </TableHeader>
                 <TableBody>
                     {destinationProperties &&
-                        Object.entries(headersMapping)
-                            .map(([importKey, mappedKey]) => {
-                                    const propertyKey = headersMapping[importKey];
-                                    const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
+                        headingsOrder.map((importKey) => {
+                                const mappedKey = headersMapping[importKey];
+                                const propertyKey = headersMapping[importKey];
+                                const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
 
-                                    const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
-                                    const originDataType = originProperty ? (originProperty.dataType === "array" && typeof originProperty.of === "object"
-                                            ? `${originProperty.dataType} - ${(originProperty.of as Property).dataType}`
-                                            : originProperty.dataType)
-                                        : undefined;
-                                    return <TableRow key={importKey} style={{ height: "90px" }}>
+                                const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
+                                const originDataType = originProperty
+                                    ? (originProperty.dataType === "array" && typeof originProperty.of === "object"
+                                        ? `${originProperty.dataType} - ${(originProperty.of as Property).dataType}`
+                                        : originProperty.dataType)
+                                    : undefined;
+                                return <TableRow key={importKey} style={{ height: "90px" }}>
+                                    <TableCell style={{ width: "20%" }}>
+                                        <Typography variant={"body2"}>{importKey}</Typography>
+                                        {originProperty && <Typography
+                                            variant={"caption"}
+                                            color={"secondary"}
+                                        >{originDataType}</Typography>}
+                                    </TableCell>
+                                    <TableCell>
+                                        <ChevronRightIcon/>
+                                    </TableCell>
+                                    <TableCell className={importKey === idColumn ? "text-center" : undefined}
+                                               style={{ width: "75%" }}>
+                                        {buildPropertyView?.({
+                                            isIdColumn: importKey === idColumn,
+                                            property,
+                                            propertyKey,
+                                            importKey
+                                        })}
+                                    </TableCell>
+                                </TableRow>;
+                            }
+                        )}
+                </TableBody>
+            </Table>
+
+            <ExpandablePanel title="Default values" initiallyExpanded={false} className={"p-4 mt-4"}>
+
+                <div className={"text-sm text-slate-500 dark:text-slate-300 font-medium ml-3.5 mb-1"}>
+                    You can select a default value for unmapped columns and empty values:
+                </div>
+                <Table style={{
+                    tableLayout: "fixed"
+                }}>
+                    <TableHeader>
+                        <TableCell header={true} style={{ width: "30%" }}>
+                            Property
+                        </TableCell>
+                        <TableCell header={true}>
+                        </TableCell>
+                        <TableCell header={true} style={{ width: "65%" }}>
+                            Default value
+                        </TableCell>
+                    </TableHeader>
+                    <TableBody>
+                        {destinationProperties &&
+                            getAllPropertyKeys(destinationProperties).map((key) => {
+                                    const property = getPropertyInPath(destinationProperties, key);
+                                    if (typeof property !== "object" || property === null) {
+                                        return null;
+                                    }
+                                    if (!["number", "string", "boolean", "map"].includes(property.dataType)) {
+                                        return null;
+                                    }
+                                    return <TableRow key={key} style={{ height: "70px" }}>
                                         <TableCell style={{ width: "20%" }}>
-                                            <Typography variant={"body2"}>{importKey}</Typography>
-                                            {originProperty && <Typography
-                                                variant={"caption"}
-                                                color={"secondary"}
-                                            >{originDataType}</Typography>}
+                                            <Typography variant={"body2"}>{key}</Typography>
                                         </TableCell>
                                         <TableCell>
                                             <ChevronRightIcon/>
                                         </TableCell>
-                                        <TableCell className={importKey === idColumn ? "text-center" : undefined}
+                                        <TableCell className={key === idColumn ? "text-center" : undefined}
                                                    style={{ width: "75%" }}>
-                                            {buildPropertyView?.({
-                                                isIdColumn: importKey === idColumn,
-                                                property,
-                                                propertyKey,
-                                                importKey
-                                            })
-                                            }
+                                            <DefaultValuesField property={property}
+                                                                defaultValue={getIn(importConfig.defaultValues, key)}
+                                                                onValueChange={(value) => {
+                                                                    const newValues = setIn(importConfig.defaultValues, key, value);
+                                                                    importConfig.setDefaultValues(newValues);
+                                                                }}/>
                                         </TableCell>
                                     </TableRow>;
                                 }
                             )}
-                </TableBody>
-            </Table>
+                    </TableBody>
+                </Table>
+            </ExpandablePanel>
         </>
     );
 }
 
+function getAllPropertyKeys(properties: PropertiesOrBuilders, currentKey?: string): string[] {
+    return Object.entries(properties).reduce((acc, [key, property]) => {
+        const accumulatedKey = currentKey ? `${currentKey}.${key}` : key;
+        if (typeof property !== "function" && property.dataType === "map" && property.properties) {
+            const childProperties = getAllPropertyKeys(property.properties, accumulatedKey);
+            return [...acc, ...childProperties];
+        }
+        return [...acc, accumulatedKey];
+    }, [] as string[]);
+}
+
 function IdSelectField({
                            idColumn,
                            headersMapping,
@@ -111,18 +180,62 @@ function IdSelectField({
             value={idColumn ?? ""}
             onChange={(event) => {
                 const value = event.target.value;
-                onChange(value === "none" ? null : value);
+                onChange(value === "__none__" ? null : value);
             }}
+            placeholder={"Autogenerate ID"}
             renderValue={(value) => {
                 return <Typography variant={"body2"}>
-                    {value !== "" ? value : "Autogenerate ID"}
+                    {value !== "__none__" ? value : "Autogenerate ID"}
                 </Typography>;
             }}
             label={"Column that will be used as ID for each document"}>
-            <SelectItem value={"none"}>Autogenerate ID</SelectItem>
+            <SelectItem value={"__none__"}>Autogenerate ID</SelectItem>
             {Object.entries(headersMapping).map(([key, value]) => {
                 return <SelectItem key={key} value={key}>{key}</SelectItem>;
             })}
         </Select>
     </div>;
 }
+
+function DefaultValuesField({
+                                property,
+                                onValueChange,
+                                defaultValue
+                            }: { property: Property, onValueChange: (value: any) => void, defaultValue?: any }) {
+    if (property.dataType === "string") {
+        return <TextField size={"small"}
+                          placeholder={"Default value"}
+                          value={defaultValue ?? ""}
+                          onChange={(event) => onValueChange(event.target.value)}/>;
+    } else if (property.dataType === "number") {
+        return <TextField size={"small"}
+                          type={"number"}
+                          value={defaultValue ?? ""}
+                          placeholder={"Default value"}
+                          onChange={(event) => onValueChange(event.target.value)}/>;
+    } else if (property.dataType === "boolean") {
+        return <BooleanSwitchWithLabel
+            value={defaultValue ?? null}
+            allowIndeterminate={true}
+            size={"small"}
+            onValueChange={(v: boolean | null) => onValueChange(v === null ? undefined : v)}
+            label={defaultValue === undefined
+                ? "Do not set value"
+                : defaultValue === true
+                    ? "Set value to true"
+                    : "Set value to false"}
+        />
+    } else if (property.dataType === "date") {
+        return <DateTimeField
+            mode={property.mode ?? "date"}
+            size={"small"}
+            value={defaultValue ?? undefined}
+            onChange={(dateValue: Date | undefined) => {
+                onValueChange(dateValue);
+            }}
+            clearable={true}
+        />
+    }
+
+    return null;
+}

package/src/components/ImportFileUpload.tsx

@@ -2,7 +2,9 @@ import { FileUpload, UploadIcon } from "@firecms/ui";
 import { convertFileToJson } from "../utils/file_to_json";
 import { useSnackbarController } from "@firecms/core";
 
-export function ImportFileUpload({ onDataAdded }: { onDataAdded: (data: object[]) => void }) {
+export function ImportFileUpload({ onDataAdded }: {
+    onDataAdded: (data: object[], propertiesOrder?: string[]) => void
+}) {
     const snackbarController = useSnackbarController();
     return <FileUpload
         accept={{
@@ -22,12 +24,18 @@ export function ImportFileUpload({ onDataAdded }: { onDataAdded: (data: object[]
         onFilesAdded={(files: File[]) => {
             if (files.length > 0) {
                 convertFileToJson(files[0])
-                    .then((jsonData) => {
-                        onDataAdded(jsonData);
+                    .then(({
+                               data,
+                               propertiesOrder
+                           }) => {
+                        onDataAdded(data, propertiesOrder);
                     })
                     .catch((error) => {
                         console.error("Error parsing file", error);
-                        snackbarController.open({ type: "error", message: error.message });
+                        snackbarController.open({
+                            type: "error",
+                            message: error.message
+                        });
                     });
             }
         }}/>

package/src/components/ImportNewPropertyFieldPreview.tsx

@@ -1,5 +1,11 @@
 import React from "react";
-import { ErrorBoundary, PropertyConfigBadge, getFieldConfig, Property, useCustomizationController } from "@firecms/core";
+import {
+    ErrorBoundary,
+    getFieldConfig,
+    Property,
+    PropertyConfigBadge,
+    useCustomizationController
+} from "@firecms/core";
 import { EditIcon, IconButton, TextField, } from "@firecms/ui";
 
 export function ImportNewPropertyFieldPreview({
@@ -49,7 +55,6 @@ export function ImportNewPropertyFieldPreview({
 
             </div>
 
-
         </div>
     </ErrorBoundary>
 }

package/src/export_import/ExportCollectionAction.tsx

@@ -36,6 +36,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                                                                                  collection: inputCollection,
                                                                                                  path: inputPath,
                                                                                                  collectionEntitiesCount,
+                                                                                                 onAnalyticsEvent,
                                                                                                  exportAllowed,
                                                                                                  notAllowedView
                                                                                              }: CollectionActionsProps<M, UserType, EntityCollection<M, any>> & {
@@ -123,6 +124,9 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
     const doDownload = useCallback(async (collection: ResolvedEntityCollection<M>,
                                           exportConfig: ExportConfig<any> | undefined) => {
 
+        onAnalyticsEvent?.("export_collection", {
+            collection: collection.path
+        });
         setDataLoading(true);
         dataSource.fetchCollection<M>({
             path,
@@ -136,6 +140,9 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                     ...collection.additionalFields?.map(field => field.key) ?? []
                 ];
                 downloadExport(data, additionalData, collection, flattenArrays, additionalHeaders, exportType, dateExportType);
+                onAnalyticsEvent?.("export_collection_success", {
+                    collection: collection.path
+                });
             })
             .catch((e) => {
                 console.error("Error loading export data", e);
@@ -143,7 +150,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
             })
             .finally(() => setDataLoading(false));
 
-    }, [dataSource, path, fetchAdditionalFields, flattenArrays, exportType, dateExportType]);
+    }, [onAnalyticsEvent, dataSource, path, fetchAdditionalFields, flattenArrays, exportType, dateExportType]);
 
     const onOkClicked = useCallback(() => {
         doDownload(collection, exportConfig);

package/src/export_import/ImportCollectionAction.tsx

@@ -32,7 +32,7 @@ import {
 } from "@firecms/ui";
 import { buildEntityPropertiesFromData } from "@firecms/schema_inference";
 import { useImportConfig } from "../hooks";
-import { convertDataToEntity, getInferenceType, getPropertiesMapping } from "../utils";
+import { convertDataToEntity, getInferenceType } from "../utils";
 import { DataNewPropertiesMapping, ImportFileUpload, ImportSaveInProgress } from "../components";
 import { ImportConfig } from "../types";
 
@@ -60,20 +60,23 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
 
     const handleClickOpen = useCallback(() => {
         setOpen(true);
+        onAnalyticsEvent?.("import_open");
         setStep("initial");
-    }, [setOpen]);
+    }, [onAnalyticsEvent]);
 
     const handleClose = useCallback(() => {
         setOpen(false);
     }, [setOpen]);
 
     const onMappingComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_mapping_complete");
         setStep("preview");
-    }, []);
+    }, [onAnalyticsEvent]);
 
     const onPreviewComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_data_save");
         setStep("import_data_saving");
-    }, []);
+    }, [onAnalyticsEvent]);
 
     const onDataAdded = async (data: object[]) => {
         importConfig.setImportData(data);
@@ -90,6 +93,7 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
             }
         }
         setTimeout(() => {
+            onAnalyticsEvent?.("import_data_added");
             setStep("mapping");
         }, 100);
         // setStep("mapping");
@@ -109,6 +113,7 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
     if (collection.collectionGroup) {
         return null;
     }
+
     return <>
 
         <Tooltip title={"Import"}>
@@ -131,17 +136,14 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
                 </>}
 
                 {step === "mapping" && <>
-                    <Typography variant={"h6"}>Map fields</Typography>
-                    <DataNewPropertiesMapping headersMapping={importConfig.headersMapping}
-                                              idColumn={importConfig.idColumn}
-                                              originProperties={importConfig.originProperties}
+                    <Typography variant={"h6"} className={"ml-3.5"}>Map fields</Typography>
+                    <DataNewPropertiesMapping importConfig={importConfig}
                                               destinationProperties={properties}
-                                              onIdPropertyChanged={(value) => importConfig.setIdColumn(value ?? undefined)}
                                               buildPropertyView={({
                                                                       isIdColumn,
                                                                       property,
                                                                       propertyKey,
-                                                                      importKey
+                                                                      importKey,
                                                                   }) => {
                                                   return <PropertyTreeSelect
                                                       selectedPropertyKey={propertyKey ?? ""}
@@ -153,6 +155,7 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
                                                       }}
                                                       onPropertySelected={(newPropertyKey) => {
 
+                                                          onAnalyticsEvent?.("import_mapping_field_updated");
                                                           const newHeadersMapping: Record<string, string | null> = Object.entries(importConfig.headersMapping)
                                                               .map(([currentImportKey, currentPropertyKey]) => {
                                                                   if (currentPropertyKey === newPropertyKey) {
@@ -253,14 +256,15 @@ function PropertyTreeSelect({
         }
 
         if (!selectedPropertyKey || !selectedProperty) {
-            return <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>;
+            return <Typography variant={"body2"} color="disabled" className={"p-4"}>Do not import this
+                property</Typography>;
         }
 
         return <PropertySelectEntry propertyKey={selectedPropertyKey}
                                     property={selectedProperty as Property}/>;
     }, [selectedProperty]);
 
-    const onSelectValueChange = useCallback((value: string) => {
+    const onSelectValueChange = (value: string) => {
         if (value === internalIDValue) {
             onIdSelected();
             onPropertySelected(null);
@@ -269,14 +273,14 @@ function PropertyTreeSelect({
         } else {
             onPropertySelected(value);
         }
-    }, []);
+    };
 
     return <Select value={isIdColumn ? internalIDValue : (selectedPropertyKey ?? undefined)}
                    onValueChange={onSelectValueChange}
                    renderValue={renderValue}>
 
         <SelectItem value={"__do_not_import"}>
-            <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>
+            <Typography variant={"body2"} color={"disabled"} className={"p-4"}>Do not import this property</Typography>
         </SelectItem>
 
         <SelectItem value={internalIDValue}>
@@ -379,8 +383,11 @@ export function ImportDataPreview<M extends Record<string, any>>({
 }) {
 
     useEffect(() => {
-        const propertiesMapping = getPropertiesMapping(importConfig.originProperties, properties);
-        const mappedData = importConfig.importData.map(d => convertDataToEntity(d, importConfig.idColumn, importConfig.headersMapping, properties, propertiesMapping, "TEMP_PATH"));
+        const mappedData = importConfig.importData.map(d => convertDataToEntity(d, importConfig.idColumn, importConfig.headersMapping, properties, "TEMP_PATH", importConfig.defaultValues));
+        console.log("Mapped data", {
+            importConfig,
+            mappedData
+        })
         importConfig.setEntities(mappedData);
     }, []);
 
@@ -400,10 +407,6 @@ export function ImportDataPreview<M extends Record<string, any>>({
         filterable={false}
         sortable={false}
         selectionController={selectionController}
-        displayedColumnIds={propertiesOrder.map(p => ({
-            key: p,
-            disabled: false
-        }))}
         properties={properties}/>
 
 }

package/src/hooks/useImportConfig.tsx

@@ -5,10 +5,12 @@ import { ImportConfig } from "../types";
 export const useImportConfig = (): ImportConfig => {
 
     const [inUse, setInUse] = useState<boolean>(false);
+    const [defaultValues, setDefaultValues] = useState<Record<string, any>>({});
     const [idColumn, setIdColumn] = useState<string | undefined>();
     const [importData, setImportData] = useState<object[]>([]);
     const [entities, setEntities] = useState<Entity<any>[]>([]);
     const [headersMapping, setHeadersMapping] = useState<Record<string, string | null>>({});
+    const [headingsOrder, setHeadingsOrder] = useState<string[]>([]);
     const [originProperties, setOriginProperties] = useState<Record<string, Property>>({});
 
     return {
@@ -20,9 +22,13 @@ export const useImportConfig = (): ImportConfig => {
         setEntities,
         importData,
         setImportData,
+        headingsOrder: (headingsOrder ?? []).length > 0 ? headingsOrder : Object.keys(headersMapping),
+        setHeadingsOrder,
         headersMapping,
         setHeadersMapping,
         originProperties,
         setOriginProperties,
+        defaultValues,
+        setDefaultValues
     };
 };

package/src/types/column_mapping.ts

@@ -22,11 +22,11 @@ export type ImportConfig = {
     originProperties: Record<string, Property>;
     setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
 
-}
+    // unmapped headings order
+    headingsOrder: string[];
+    setHeadingsOrder: React.Dispatch<React.SetStateAction<string[]>>;
+
+    defaultValues: Record<string, any>;
+    setDefaultValues: React.Dispatch<React.SetStateAction<Record<string, any>>>;
 
-export type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
 }