@firecms/data_import_export 3.0.0-canary.5 → 3.0.0-canary.51

package/src/export_import/ExportCollectionAction.tsx

@@ -28,7 +28,7 @@ import {
     Tooltip,
     Typography,
 } from "@firecms/ui";
-import { downloadExport } from "./export";
+import { downloadEntitiesExport } from "./export";
 
 const DOCS_LIMIT = 500;
 
@@ -36,11 +36,13 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                                                                                  collection: inputCollection,
                                                                                                  path: inputPath,
                                                                                                  collectionEntitiesCount,
+                                                                                                 onAnalyticsEvent,
                                                                                                  exportAllowed,
                                                                                                  notAllowedView
                                                                                              }: CollectionActionsProps<M, UserType, EntityCollection<M, any>> & {
     exportAllowed?: (props: { collectionEntitiesCount: number, path: string, collection: EntityCollection }) => boolean;
     notAllowedView?: React.ReactNode;
+    onAnalyticsEvent?: (event: string, params?: any) => void;
 }) {
 
     const customizationController = useCustomizationController();
@@ -122,6 +124,9 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
     const doDownload = useCallback(async (collection: ResolvedEntityCollection<M>,
                                           exportConfig: ExportConfig<any> | undefined) => {
 
+        onAnalyticsEvent?.("export_collection", {
+            collection: collection.path
+        });
         setDataLoading(true);
         dataSource.fetchCollection<M>({
             path,
@@ -134,7 +139,10 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                     ...exportConfig?.additionalFields?.map(column => column.key) ?? [],
                     ...collection.additionalFields?.map(field => field.key) ?? []
                 ];
-                downloadExport(data, additionalData, collection, flattenArrays, additionalHeaders, exportType, dateExportType);
+                downloadEntitiesExport(data, additionalData, collection.properties, collection.propertiesOrder, collection.name, flattenArrays, additionalHeaders, exportType, dateExportType);
+                onAnalyticsEvent?.("export_collection_success", {
+                    collection: collection.path
+                });
             })
             .catch((e) => {
                 console.error("Error loading export data", e);
@@ -142,7 +150,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
             })
             .finally(() => setDataLoading(false));
 
-    }, [dataSource, path, fetchAdditionalFields, flattenArrays, exportType, dateExportType]);
+    }, [onAnalyticsEvent, dataSource, path, fetchAdditionalFields, flattenArrays, exportType, dateExportType]);
 
     const onOkClicked = useCallback(() => {
         doDownload(collection, exportConfig);
@@ -183,7 +191,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                    onChange={() => setExportType("csv")}
                                    className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
                             <label htmlFor="radio-csv"
-                                   className="p-2 text-sm font-medium text-gray-900 dark:text-gray-300">CSV</label>
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">CSV</label>
                         </div>
                         <div className="flex items-center">
                             <input id="radio-json" type="radio" value="json" name="exportType"
@@ -191,7 +199,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                    onChange={() => setExportType("json")}
                                    className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
                             <label htmlFor="radio-json"
-                                   className="p-2 text-sm font-medium text-gray-900 dark:text-gray-300">JSON</label>
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">JSON</label>
                         </div>
                     </div>
 
@@ -202,7 +210,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                    onChange={() => setDateExportType("timestamp")}
                                    className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
                             <label htmlFor="radio-timestamp"
-                                   className="p-2 text-sm font-medium text-gray-900 dark:text-gray-300">Dates as
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">Dates as
                                 timestamps ({dateRef.current.getTime()})</label>
                         </div>
                         <div className="flex items-center">
@@ -211,7 +219,7 @@ export function ExportCollectionAction<M extends Record<string, any>, UserType e
                                    onChange={() => setDateExportType("string")}
                                    className={cn(focusedMixin, "w-4 text-primary-dark bg-gray-100 border-gray-300 dark:bg-gray-700 dark:border-gray-600")}/>
                             <label htmlFor="radio-string"
-                                   className="p-2 text-sm font-medium text-gray-900 dark:text-gray-300">Dates as
+                                   className="p-2 text-sm font-medium text-gray-900 dark:text-slate-300">Dates as
                                 strings ({dateRef.current.toISOString()})</label>
                         </div>
                     </div>

package/src/export_import/ImportCollectionAction.tsx

@@ -2,13 +2,15 @@ import React, { useCallback, useEffect } from "react";
 import {
     CollectionActionsProps,
     EntityCollectionTable,
-    PropertyConfigBadge,
     getFieldConfig,
     getPropertiesWithPropertiesOrder,
     getPropertyInPath,
+    PropertiesOrBuilders,
     Property,
+    PropertyConfigBadge,
     resolveCollection,
     ResolvedProperties,
+    slugify,
     useCustomizationController,
     User,
     useSelectionController,
@@ -30,7 +32,7 @@ import {
 } from "@firecms/ui";
 import { buildEntityPropertiesFromData } from "@firecms/schema_inference";
 import { useImportConfig } from "../hooks";
-import { convertDataToEntity, getInferenceType, getPropertiesMapping } from "../utils";
+import { convertDataToEntity, getInferenceType } from "../utils";
 import { DataNewPropertiesMapping, ImportFileUpload, ImportSaveInProgress } from "../components";
 import { ImportConfig } from "../types";
 
@@ -40,8 +42,12 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
                                                                                                  collection,
                                                                                                  path,
                                                                                                  collectionEntitiesCount,
-                                                                                             }: CollectionActionsProps<M, UserType>
+                                                                                                 onAnalyticsEvent
+                                                                                             }: CollectionActionsProps<M, UserType> & {
+                                                                                                 onAnalyticsEvent?: (event: string, params?: any) => void;
+                                                                                             }
 ) {
+
     const customizationController = useCustomizationController();
 
     const snackbarController = useSnackbarController();
@@ -54,20 +60,23 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
 
     const handleClickOpen = useCallback(() => {
         setOpen(true);
+        onAnalyticsEvent?.("import_open");
         setStep("initial");
-    }, [setOpen]);
+    }, [onAnalyticsEvent]);
 
     const handleClose = useCallback(() => {
         setOpen(false);
     }, [setOpen]);
 
     const onMappingComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_mapping_complete");
         setStep("preview");
-    }, []);
+    }, [onAnalyticsEvent]);
 
     const onPreviewComplete = useCallback(() => {
+        onAnalyticsEvent?.("import_data_save");
         setStep("import_data_saving");
-    }, []);
+    }, [onAnalyticsEvent]);
 
     const onDataAdded = async (data: object[]) => {
         importConfig.setImportData(data);
@@ -76,15 +85,15 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
             const originProperties = await buildEntityPropertiesFromData(data, getInferenceType);
             importConfig.setOriginProperties(originProperties);
 
-            const headersMapping = buildHeadersMappingFromData(data);
+            const headersMapping = buildHeadersMappingFromData(data, collection?.properties);
             importConfig.setHeadersMapping(headersMapping);
             const firstKey = Object.keys(headersMapping)?.[0];
             if (firstKey?.includes("id") || firstKey?.includes("key")) {
-                const idColumn = firstKey;
-                importConfig.setIdColumn(idColumn);
+                importConfig.setIdColumn(firstKey);
             }
         }
         setTimeout(() => {
+            onAnalyticsEvent?.("import_data_added");
             setStep("mapping");
         }, 100);
         // setStep("mapping");
@@ -104,6 +113,7 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
     if (collection.collectionGroup) {
         return null;
     }
+
     return <>
 
         <Tooltip title={"Import"}>
@@ -126,17 +136,14 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
                 </>}
 
                 {step === "mapping" && <>
-                    <Typography variant={"h6"}>Map fields</Typography>
-                    <DataNewPropertiesMapping headersMapping={importConfig.headersMapping}
-                                              idColumn={importConfig.idColumn}
-                                              originProperties={importConfig.originProperties}
+                    <Typography variant={"h6"} className={"ml-3.5"}>Map fields</Typography>
+                    <DataNewPropertiesMapping importConfig={importConfig}
                                               destinationProperties={properties}
-                                              onIdPropertyChanged={(value) => importConfig.setIdColumn(value)}
                                               buildPropertyView={({
                                                                       isIdColumn,
                                                                       property,
                                                                       propertyKey,
-                                                                      importKey
+                                                                      importKey,
                                                                   }) => {
                                                   return <PropertyTreeSelect
                                                       selectedPropertyKey={propertyKey ?? ""}
@@ -148,6 +155,7 @@ export function ImportCollectionAction<M extends Record<string, any>, UserType e
                                                       }}
                                                       onPropertySelected={(newPropertyKey) => {
 
+                                                          onAnalyticsEvent?.("import_mapping_field_updated");
                                                           const newHeadersMapping: Record<string, string | null> = Object.entries(importConfig.headersMapping)
                                                               .map(([currentImportKey, currentPropertyKey]) => {
                                                                   if (currentPropertyKey === newPropertyKey) {
@@ -248,30 +256,31 @@ function PropertyTreeSelect({
         }
 
         if (!selectedPropertyKey || !selectedProperty) {
-            return <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>;
+            return <Typography variant={"body2"} color="disabled" className={"p-4"}>Do not import this
+                property</Typography>;
         }
 
         return <PropertySelectEntry propertyKey={selectedPropertyKey}
                                     property={selectedProperty as Property}/>;
     }, [selectedProperty]);
 
-    const onSelectValueChange = useCallback((value: string) => {
+    const onSelectValueChange = (value: string) => {
         if (value === internalIDValue) {
             onIdSelected();
             onPropertySelected(null);
-        } else if (value === "") {
+        } else if (value === "__do_not_import") {
             onPropertySelected(null);
         } else {
             onPropertySelected(value);
         }
-    }, []);
+    };
 
     return <Select value={isIdColumn ? internalIDValue : (selectedPropertyKey ?? undefined)}
                    onValueChange={onSelectValueChange}
                    renderValue={renderValue}>
 
-        <SelectItem value={""}>
-            <Typography variant={"body2"} className={"p-4"}>Do not import this property</Typography>
+        <SelectItem value={"__do_not_import"}>
+            <Typography variant={"body2"} color={"disabled"} className={"p-4"}>Do not import this property</Typography>
         </SelectItem>
 
         <SelectItem value={internalIDValue}>
@@ -370,12 +379,11 @@ export function ImportDataPreview<M extends Record<string, any>>({
                                                                  }: {
     importConfig: ImportConfig,
     properties: ResolvedProperties<M>,
-    propertiesOrder: Extract<keyof M, string>[]
+    propertiesOrder: Extract<keyof M, string>[],
 }) {
 
     useEffect(() => {
-        const propertiesMapping = getPropertiesMapping(importConfig.originProperties, properties);
-        const mappedData = importConfig.importData.map(d => convertDataToEntity(d, importConfig.idColumn, importConfig.headersMapping, properties, propertiesMapping, "TEMP_PATH"));
+        const mappedData = importConfig.importData.map(d => convertDataToEntity(d, importConfig.idColumn, importConfig.headersMapping, properties, "TEMP_PATH", importConfig.defaultValues));
         importConfig.setEntities(mappedData);
     }, []);
 
@@ -391,30 +399,43 @@ export function ImportDataPreview<M extends Record<string, any>>({
             dataLoading: false,
             noMoreToLoad: false
         }}
+        enablePopupIcon={false}
         endAdornment={<div className={"h-12"}/>}
         filterable={false}
         sortable={false}
         selectionController={selectionController}
-        displayedColumnIds={propertiesOrder.map(p => ({
-            key: p,
-            disabled: false
-        }))}
         properties={properties}/>
 
 }
 
-function buildHeadersMappingFromData(objArr: object[]) {
+function buildHeadersMappingFromData(objArr: object[], properties?: PropertiesOrBuilders<any>) {
     const headersMapping: Record<string, string> = {};
     objArr.filter(Boolean).forEach((obj) => {
         Object.keys(obj).forEach((key) => {
             // @ts-ignore
             const child = obj[key];
             if (typeof child === "object" && !Array.isArray(child)) {
-                Object.entries(buildHeadersMappingFromData([child])).forEach(([subKey, mapping]) => {
+                const childProperty = properties?.[key];
+                const childProperties = childProperty && "properties" in childProperty ? childProperty.properties : undefined;
+                const childHeadersMapping = buildHeadersMappingFromData([child], childProperties);
+                Object.entries(childHeadersMapping).forEach(([subKey, mapping]) => {
                     headersMapping[`${key}.${subKey}`] = `${key}.${mapping}`;
                 });
             }
-            headersMapping[key] = key;
+
+            if (!properties) {
+                headersMapping[key] = key;
+            } else if (key in properties) {
+                headersMapping[key] = key;
+            } else {
+                const slug = slugify(key);
+                if (slug in properties) {
+                    headersMapping[key] = slug;
+                } else {
+                    headersMapping[key] = key;
+                }
+            }
+
         });
     });
     return headersMapping;

package/src/export_import/export.ts

@@ -4,7 +4,6 @@ import {
     EntityReference,
     getArrayValuesCount,
     getValueInPath,
-    ResolvedEntityCollection,
     ResolvedProperties,
     ResolvedProperty
 } from "@firecms/core";
@@ -14,37 +13,43 @@ interface Header {
     label: string;
 }
 
-export function downloadExport<M extends Record<string, any>>(data: Entity<M>[],
-                                                              additionalData: Record<string, any>[] | undefined,
-                                                              collection: ResolvedEntityCollection<M>,
-                                                              flattenArrays: boolean,
-                                                              additionalHeaders: string[] | undefined,
-                                                              exportType: "csv" | "json",
-                                                              dateExportType: "timestamp" | "string"
+export function downloadEntitiesExport<M extends Record<string, any>>(data: Entity<M>[],
+                                                                      additionalData: Record<string, any>[] | undefined,
+                                                                      properties: ResolvedProperties<M>,
+                                                                      propertiesOrder: string[] | undefined,
+                                                                      name: string,
+                                                                      flattenArrays: boolean,
+                                                                      additionalHeaders: string[] | undefined,
+                                                                      exportType: "csv" | "json",
+                                                                      dateExportType: "timestamp" | "string"
 ) {
 
-    console.debug("Downloading export", { dataLength: data.length, collection, exportType, dateExportType });
-    const properties = collection.properties;
+    console.debug("Downloading export", {
+        dataLength: data.length,
+        properties,
+        exportType,
+        dateExportType
+    });
 
     if (exportType === "csv") {
         const arrayValuesCount = flattenArrays ? getArrayValuesCount(data.map(d => d.values)) : {};
-        const headers = getExportHeaders(properties, additionalHeaders, arrayValuesCount);
-        const exportableData = getCSVExportableData(data, additionalData, properties, headers, dateExportType);
+        const headers = getExportHeaders(properties, propertiesOrder, additionalHeaders, arrayValuesCount);
+        const exportableData = getEntityCSVExportableData(data, additionalData, properties, headers, dateExportType);
         const headersData = entryToCSVRow(headers.map(h => h.label));
         const csvData = exportableData.map(entry => entryToCSVRow(entry));
-        downloadBlob([headersData, ...csvData], `${collection.name}.csv`, "text/csv");
+        downloadBlob([headersData, ...csvData], `${name}.csv`, "text/csv");
     } else {
-        const exportableData = getJsonExportableData(data, additionalData, properties, dateExportType);
+        const exportableData = getEntityJsonExportableData(data, additionalData, properties, dateExportType);
         const json = JSON.stringify(exportableData, null, 2);
-        downloadBlob([json], `${collection.name}.json`, "application/json");
+        downloadBlob([json], `${name}.json`, "application/json");
     }
 }
 
-export function getCSVExportableData(data: Entity<any>[],
-                                     additionalData: Record<string, any>[] | undefined,
-                                     properties: ResolvedProperties,
-                                     headers: Header[],
-                                     dateExportType: "timestamp" | "string"
+export function getEntityCSVExportableData(data: Entity<any>[],
+                                           additionalData: Record<string, any>[] | undefined,
+                                           properties: ResolvedProperties,
+                                           headers: Header[],
+                                           dateExportType: "timestamp" | "string"
 ) {
 
     const mergedData: any[] = data.map(e => ({
@@ -63,10 +68,10 @@ export function getCSVExportableData(data: Entity<any>[],
     });
 }
 
-export function getJsonExportableData(data: Entity<any>[],
-                                      additionalData: Record<string, any>[] | undefined,
-                                      properties: ResolvedProperties,
-                                      dateExportType: "timestamp" | "string"
+export function getEntityJsonExportableData(data: Entity<any>[],
+                                            additionalData: Record<string, any>[] | undefined,
+                                            properties: ResolvedProperties,
+                                            dateExportType: "timestamp" | "string"
 ) {
 
     const mergedData: any[] = data.map(e => ({
@@ -84,13 +89,22 @@ export function getJsonExportableData(data: Entity<any>[],
 }
 
 function getExportHeaders<M extends Record<string, any>>(properties: ResolvedProperties<M>,
+                                                         propertiesOrder: string[] | undefined,
                                                          additionalHeaders: string[] | undefined,
                                                          arrayValuesCount?: ArrayValuesCount): Header[] {
 
     const headers: Header[] = [
-        { label: "id", key: "id" },
-        ...Object.entries(properties)
-            .flatMap(([childKey, property]) => {
+        {
+            label: "id",
+            key: "id"
+        },
+        ...(propertiesOrder ?? Object.keys(properties))
+            .flatMap((childKey) => {
+                const property = properties[childKey];
+                if(!property) {
+                    console.warn("Property not found", childKey, properties);
+                    return [];
+                }
                 if (arrayValuesCount && arrayValuesCount[childKey] > 1) {
                     return Array.from({ length: arrayValuesCount[childKey] },
                         (_, i) => getHeaders(property as ResolvedProperty, `${childKey}[${i}]`, ""))
@@ -102,7 +116,10 @@ function getExportHeaders<M extends Record<string, any>>(properties: ResolvedPro
     ];
 
     if (additionalHeaders) {
-        headers.push(...additionalHeaders.map(h => ({ label: h, key: h })));
+        headers.push(...additionalHeaders.map(h => ({
+            label: h,
+            key: h
+        })));
     }
 
     return headers;
@@ -121,7 +138,10 @@ function getHeaders(property: ResolvedProperty, propertyKey: string, prefix = ""
             .map(([childKey, p]) => getHeaders(p, childKey, currentKey))
             .flat();
     } else {
-        return [{ label: currentKey, key: currentKey }];
+        return [{
+            label: currentKey,
+            key: currentKey
+        }];
     }
 }
 
@@ -149,7 +169,7 @@ function processValueForExport(inputValue: any,
         } else {
             value = inputValue;
         }
-    } else if (property.dataType === "reference" && inputValue.isEntityReference && inputValue.isEntityReference()) {
+    } else if (property.dataType === "reference" && inputValue && inputValue.isEntityReference && inputValue.isEntityReference()) {
         const ref = inputValue ? inputValue as EntityReference : undefined;
         value = ref ? ref.pathWithId : null;
     } else if (property.dataType === "date" && inputValue instanceof Date) {

package/src/export_import/index.ts

@@ -0,0 +1,4 @@
+export * from "./export";
+export * from "./BasicExportAction";
+export * from "./ExportCollectionAction";
+export * from "./ImportCollectionAction";

package/src/hooks/useImportConfig.tsx

@@ -5,10 +5,12 @@ import { ImportConfig } from "../types";
 export const useImportConfig = (): ImportConfig => {
 
     const [inUse, setInUse] = useState<boolean>(false);
+    const [defaultValues, setDefaultValues] = useState<Record<string, any>>({});
     const [idColumn, setIdColumn] = useState<string | undefined>();
     const [importData, setImportData] = useState<object[]>([]);
     const [entities, setEntities] = useState<Entity<any>[]>([]);
     const [headersMapping, setHeadersMapping] = useState<Record<string, string | null>>({});
+    const [headingsOrder, setHeadingsOrder] = useState<string[]>([]);
     const [originProperties, setOriginProperties] = useState<Record<string, Property>>({});
 
     return {
@@ -20,9 +22,13 @@ export const useImportConfig = (): ImportConfig => {
         setEntities,
         importData,
         setImportData,
+        headingsOrder: (headingsOrder ?? []).length > 0 ? headingsOrder : Object.keys(headersMapping),
+        setHeadingsOrder,
         headersMapping,
         setHeadersMapping,
         originProperties,
         setOriginProperties,
+        defaultValues,
+        setDefaultValues
     };
 };

package/src/index.ts

@@ -3,3 +3,4 @@ export * from "./components";
 export * from "./types";
 export * from "./utils";
 export * from "./hooks";
+export * from "./export_import";

package/src/types/column_mapping.ts

@@ -22,11 +22,11 @@ export type ImportConfig = {
     originProperties: Record<string, Property>;
     setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
 
-}
+    // unmapped headings order
+    headingsOrder: string[];
+    setHeadingsOrder: React.Dispatch<React.SetStateAction<string[]>>;
+
+    defaultValues: Record<string, any>;
+    setDefaultValues: React.Dispatch<React.SetStateAction<Record<string, any>>>;
 
-export type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
 }

package/src/useImportExportPlugin.tsx

@@ -9,8 +9,8 @@ import { ExportCollectionAction } from "./export_import/ExportCollectionAction";
 export function useImportExportPlugin(props?: ImportExportPluginProps): FireCMSPlugin<any, any, any, ImportExportPluginProps> {
 
     return useMemo(() => ({
-        name: "Import/Export",
-        collections: {
+        key: "import_export",
+        collectionView: {
             CollectionActions: [ImportCollectionAction, ExportCollectionAction],
             collectionActionsProps: props
         }
@@ -19,6 +19,7 @@ export function useImportExportPlugin(props?: ImportExportPluginProps): FireCMSP
 
 export type ImportExportPluginProps = {
     exportAllowed?: (props: ExportAllowedParams) => boolean;
-    notAllowedView: React.ReactNode;
+    notAllowedView?: React.ReactNode;
+    onAnalyticsEvent?: (event: string, params?: any) => void;
 }
 export type ExportAllowedParams = { collectionEntitiesCount: number, path: string, collection: EntityCollection };