@firecms/data_import_export 3.0.0-alpha.38

package/src/export_import/export.ts

@@ -0,0 +1,198 @@
+import {
+    ArrayValuesCount,
+    Entity,
+    EntityReference,
+    getArrayValuesCount,
+    getValueInPath,
+    ResolvedEntityCollection,
+    ResolvedProperties,
+    ResolvedProperty
+} from "@firecms/core";
+
+interface Header {
+    key: string;
+    label: string;
+}
+
+export function downloadExport<M extends Record<string, any>>(data: Entity<M>[],
+                                                              additionalData: Record<string, any>[] | undefined,
+                                                              collection: ResolvedEntityCollection<M>,
+                                                              flattenArrays: boolean,
+                                                              additionalHeaders: string[] | undefined,
+                                                              exportType: "csv" | "json",
+                                                              dateExportType: "timestamp" | "string"
+) {
+    const properties = collection.properties;
+
+    if (exportType === "csv") {
+        const arrayValuesCount = flattenArrays ? getArrayValuesCount(data.map(d => d.values)) : {};
+        const headers = getExportHeaders(properties, additionalHeaders, arrayValuesCount);
+        const exportableData = getCSVExportableData(data, additionalData, properties, headers, dateExportType);
+        const headersData = entryToCSVRow(headers.map(h => h.label));
+        const csvData = exportableData.map(entry => entryToCSVRow(entry));
+        downloadBlob([headersData, ...csvData], `${collection.name}.csv`, "text/csv");
+    } else {
+        const exportableData = getJsonExportableData(data, additionalData, properties, dateExportType);
+        const json = JSON.stringify(exportableData, null, 2);
+        downloadBlob([json], `${collection.name}.json`, "application/json");
+    }
+}
+
+export function getCSVExportableData(data: Entity<any>[],
+                                     additionalData: Record<string, any>[] | undefined,
+                                     properties: ResolvedProperties,
+                                     headers: Header[],
+                                     dateExportType: "timestamp" | "string"
+) {
+
+    const mergedData: any[] = data.map(e => ({
+        id: e.id,
+        ...processValuesForExport(e.values, properties, "csv", dateExportType)
+    }));
+
+    if (additionalData) {
+        additionalData.forEach((additional, index) => {
+            mergedData[index] = { ...mergedData[index], ...additional };
+        });
+    }
+
+    return mergedData && mergedData.map((entry) => {
+        return headers.map((header) => getValueInPath(entry, header.key));
+    });
+}
+
+export function getJsonExportableData(data: Entity<any>[],
+                                      additionalData: Record<string, any>[] | undefined,
+                                      properties: ResolvedProperties,
+                                      dateExportType: "timestamp" | "string"
+) {
+
+    const mergedData: any[] = data.map(e => ({
+        id: e.id,
+        ...processValuesForExport(e.values, properties, "json", dateExportType)
+    }));
+
+    if (additionalData) {
+        additionalData.forEach((additional, index) => {
+            mergedData[index] = { ...mergedData[index], ...additional };
+        });
+    }
+
+    return mergedData;
+}
+
+function getExportHeaders<M extends Record<string, any>>(properties: ResolvedProperties<M>,
+                                                         additionalHeaders: string[] | undefined,
+                                                         arrayValuesCount?: ArrayValuesCount): Header[] {
+
+    const headers: Header[] = [
+        { label: "id", key: "id" },
+        ...Object.entries(properties)
+            .flatMap(([childKey, property]) => {
+                if (arrayValuesCount && arrayValuesCount[childKey] > 1) {
+                    return Array.from({ length: arrayValuesCount[childKey] },
+                        (_, i) => getHeaders(property as ResolvedProperty, `${childKey}[${i}]`, ""))
+                        .flat();
+                } else {
+                    return getHeaders(property as ResolvedProperty, childKey, "");
+                }
+            })
+    ];
+
+    if (additionalHeaders) {
+        headers.push(...additionalHeaders.map(h => ({ label: h, key: h })));
+    }
+
+    return headers;
+}
+
+/**
+ * Get headers for property. There could be more than one header per property
+ * @param property
+ * @param propertyKey
+ * @param prefix
+ */
+function getHeaders(property: ResolvedProperty, propertyKey: string, prefix = ""): Header[] {
+    const currentKey = prefix ? `${prefix}.${propertyKey}` : propertyKey;
+    if (property.dataType === "map" && property.properties) {
+        return Object.entries(property.properties)
+            .map(([childKey, p]) => getHeaders(p, childKey, currentKey))
+            .flat();
+    } else {
+        return [{ label: currentKey, key: currentKey }];
+    }
+}
+
+function processValueForExport(inputValue: any,
+                               property: ResolvedProperty,
+                               exportType: "csv" | "json",
+                               dateExportType: "timestamp" | "string"
+): any {
+
+    let value;
+    if (property.dataType === "map" && property.properties) {
+        value = processValuesForExport(inputValue, property.properties as ResolvedProperties, exportType, dateExportType);
+    } else if (property.dataType === "array") {
+        if (property.of && Array.isArray(inputValue)) {
+            if (Array.isArray(property.of)) {
+                value = property.of.map((p, i) => processValueForExport(inputValue[i], p, exportType, dateExportType));
+            } else if (property.of.dataType === "map") {
+                value = exportType === "csv"
+                    ? inputValue.map((e) => JSON.stringify(e))
+                    : inputValue.map((e) => processValueForExport(e, property.of as ResolvedProperty, exportType, dateExportType));
+                ;
+            } else {
+                value = inputValue.map((e) => processValueForExport(e, property.of as ResolvedProperty, exportType, dateExportType));
+            }
+        } else {
+            value = inputValue;
+        }
+    } else if (property.dataType === "reference" && inputValue instanceof EntityReference) {
+        const ref = inputValue ? inputValue as EntityReference : undefined;
+        value = ref ? ref.pathWithId : null;
+    } else if (property.dataType === "date" && inputValue instanceof Date) {
+        value = inputValue ? (dateExportType === "timestamp" ? inputValue.getTime() : inputValue.toISOString()) : null;
+    } else {
+        value = inputValue;
+    }
+
+    return value;
+}
+
+function processValuesForExport<M extends Record<string, any>>
+(inputValues: Record<keyof M, any>,
+ properties: ResolvedProperties<M>,
+ exportType: "csv" | "json",
+ dateExportType: "timestamp" | "string"
+): Record<keyof M, any> {
+    const updatedValues = Object.entries(properties)
+        .map(([key, property]) => {
+            const inputValue = inputValues && (inputValues)[key];
+            const updatedValue = processValueForExport(inputValue, property as ResolvedProperty, exportType, dateExportType);
+            if (updatedValue === undefined) return {};
+            return ({ [key]: updatedValue });
+        })
+        .reduce((a, b) => ({ ...a, ...b }), {}) as Record<keyof M, any>;
+    return { ...inputValues, ...updatedValues };
+}
+
+function entryToCSVRow(entry: any[]) {
+    return entry
+        .map((v: any) => {
+            if (v === null || v === undefined) return "";
+            if (Array.isArray(v))
+                return "\"" + JSON.stringify(v).replaceAll("\"", "\\\"") + "\"";
+            const s = String(v);
+            return "\"" + s.replaceAll("\"", "\"\"") + "\"";
+        })
+        .join(",") + "\r\n";
+}
+
+export function downloadBlob(content: BlobPart[], filename: string, contentType: string) {
+    const blob = new Blob(content, { type: contentType });
+    const url = URL.createObjectURL(blob);
+    const pom = document.createElement("a");
+    pom.href = url;
+    pom.setAttribute("download", filename);
+    pom.click();
+}

package/src/hooks/index.ts

@@ -0,0 +1 @@
+export * from "./useImportConfig";

package/src/hooks/useImportConfig.tsx

@@ -0,0 +1,28 @@
+import { useCallback, useState } from "react";
+import { DataType, Entity, getPropertyInPath, Properties, Property } from "@firecms/core";
+import { DataTypeMapping, ImportConfig } from "../types";
+
+export const useImportConfig = (): ImportConfig => {
+
+    const [inUse, setInUse] = useState<boolean>(false);
+    const [idColumn, setIdColumn] = useState<string | undefined>();
+    const [importData, setImportData] = useState<object[]>([]);
+    const [entities, setEntities] = useState<Entity<any>[]>([]);
+    const [headersMapping, setHeadersMapping] = useState<Record<string, string | null>>({});
+    const [originProperties, setOriginProperties] = useState<Record<string, Property>>({});
+
+    return {
+        inUse,
+        setInUse,
+        idColumn,
+        setIdColumn,
+        entities,
+        setEntities,
+        importData,
+        setImportData,
+        headersMapping,
+        setHeadersMapping,
+        originProperties,
+        setOriginProperties,
+    };
+};

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from "./useImportExportPlugin";
+export * from "./components";
+export * from "./types";
+export * from "./utils";
+export * from "./hooks";

package/src/types/column_mapping.ts

@@ -0,0 +1,32 @@
+import React from "react";
+import { DataType, Entity, Property } from "@firecms/core";
+
+export type ImportConfig = {
+
+    inUse: boolean;
+    setInUse: React.Dispatch<React.SetStateAction<boolean>>;
+
+    idColumn: string | undefined;
+    setIdColumn: React.Dispatch<React.SetStateAction<string | undefined>>;
+
+    importData: object[];
+    setImportData: React.Dispatch<React.SetStateAction<object[]>>;
+
+    entities: Entity<any>[];
+    setEntities: React.Dispatch<React.SetStateAction<Entity<any>[]>>;
+
+    // mapping of the column name in the import file to the property key in the data model
+    headersMapping: Record<string, string | null>;
+    setHeadersMapping: React.Dispatch<React.SetStateAction<Record<string, string | null>>>;
+
+    originProperties: Record<string, Property>;
+    setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
+
+}
+
+export type DataTypeMapping = {
+    from: DataType;
+    fromSubtype?: DataType;
+    to: DataType;
+    toSubtype?: DataType;
+}

package/src/types/index.ts

@@ -0,0 +1 @@
+export * from "./column_mapping";

package/src/useImportExportPlugin.tsx

@@ -0,0 +1,23 @@
+import React from "react";
+import { EntityCollection, FireCMSPlugin } from "@firecms/core";
+import { ImportCollectionAction } from "./export_import/ImportCollectionAction";
+import { ExportCollectionAction } from "./export_import/ExportCollectionAction";
+
+/**
+ *
+ */
+export function useImportExportPlugin(props?: ImportExportPluginProps): FireCMSPlugin<any, any, any, ImportExportPluginProps> {
+
+    return {
+        name: "Import/Export",
+        collections: {
+            CollectionActions: [ImportCollectionAction, ExportCollectionAction],
+            collectionActionsProps: props
+        }
+    };
+}
+
+export type ImportExportPluginProps = {
+    exportAllowed?: (props: { collectionEntitiesCount: number, path: string, collection: EntityCollection }) => boolean;
+    notAllowedView: React.ReactNode;
+}

package/src/utils/data.ts

@@ -0,0 +1,210 @@
+import { DataType, Entity, EntityReference, getPropertyInPath, Properties } from "@firecms/core";
+import { unflattenObject } from "./file_to_json";
+
+type DataTypeMapping = {
+    from: DataType;
+    fromSubtype?: DataType;
+    to: DataType;
+    toSubtype?: DataType;
+}
+
+export function convertDataToEntity(data: Record<any, any>,
+                                    idColumn: string | undefined,
+                                    headersMapping: Record<string, string | null>,
+                                    properties: Properties,
+                                    propertiesMapping: Record<string, DataTypeMapping>,
+                                    path: string): Entity<any> {
+    const flatObject = flattenEntry(data);
+    if (idColumn)
+        delete flatObject[idColumn];
+    const mappedKeysObject = Object.entries(flatObject)
+        .map(([key, value]) => {
+            const mappedKey = headersMapping[key] ?? key;
+
+            const mappedProperty = getPropertyInPath(properties, mappedKey);
+            if (!mappedProperty)
+                return {};
+
+            const propertyMapping = propertiesMapping[mappedKey];
+            let valueResult = value;
+            if (propertyMapping) {
+                valueResult = processValueMapping(value, propertyMapping);
+            }
+            return ({
+                [mappedKey]: valueResult
+            });
+        })
+        .reduce((acc, curr) => ({ ...acc, ...curr }), {});
+    const values = unflattenObject(mappedKeysObject);
+    return {
+        id: idColumn ? data[idColumn] : undefined,
+        values,
+        path
+    };
+}
+
+export function flattenEntry(obj: any, parent = ""): any {
+    return Object.keys(obj).reduce((acc, key) => {
+        const prefixedKey = parent ? `${parent}.${key}` : key;
+
+        if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
+            Object.assign(acc, flattenEntry(obj[key], prefixedKey));
+        } else {
+            // @ts-ignore
+            acc[prefixedKey] = obj[key];
+        }
+
+        return acc;
+    }, {});
+}
+
+// export function convertDataEntryValue({
+//                                           key,
+//                                           fullKey,
+//                                           value,
+//                                           idColumn,
+//                                           headersMapping,
+//                                           properties,
+//                                           propertiesMapping
+//                                       }: {
+//     key?: string,
+//     fullKey: string,
+//     value: any,
+//     idColumn?: string,
+//     headersMapping: Record<string, string | null>,
+//     properties: Properties,
+//     propertiesMapping: Record<string, DataTypeMapping>
+// }): any {
+//
+//     if (value === undefined) return value;
+//     if (value === null) return value;
+//
+//     if (Array.isArray(value)) {
+//         const mappedKey = headersMapping[fullKey] || fullKey;
+//         const valueMapping = propertiesMapping[mappedKey];
+//         return processValueMapping(value, valueMapping);
+//         // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
+//     } else if (typeof value === "object") {
+//         return convertDataObjectValue({
+//             key,
+//             fullKey,
+//             value,
+//             idColumn,
+//             headersMapping,
+//             properties,
+//             propertiesMapping
+//         });
+//     } else {
+//         const mappedKey = headersMapping[fullKey] || fullKey;
+//         const valueMapping = propertiesMapping[mappedKey];
+//         return processValueMapping(value, valueMapping);
+//     }
+// }
+
+// export function convertDataObjectValue({
+//                                            key,
+//                                            fullKey,
+//                                            value,
+//                                            idColumn,
+//                                            headersMapping,
+//                                            properties,
+//                                            propertiesMapping
+//                                        }: {
+//     key?: string,
+//     fullKey?: string,
+//     value: object,
+//     idColumn?: string,
+//     headersMapping: Record<string, string | null>,
+//     properties: Properties,
+//     propertiesMapping: Record<string, DataTypeMapping>
+// }): object {
+//
+//     if (value instanceof Date) {
+//         const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
+//         const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
+//         return processValueMapping(value, valueMapping);
+//     }
+//
+//     return Object.entries(value)
+//         .map(([childKey, childValue]) => {
+//             const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
+//             const mappedKey = headersMapping[childFullKey] ?? childFullKey;
+//             const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
+//             const property = getPropertyInPath(properties, mappedKey);
+//             // if (!property) {
+//             //     console.log("ppp", { properties, mappedKey,  })
+//             //     return {};
+//             // }
+//             return ({
+//                 [mappedKeyLastPart]: convertDataEntryValue({
+//                     key: childKey,
+//                     fullKey: childFullKey,
+//                     value: childValue,
+//                     idColumn,
+//                     headersMapping,
+//                     properties,
+//                     propertiesMapping
+//                 })
+//             });
+//         })
+//         .reduce((acc, curr) => ({ ...acc, ...curr }), {});
+// }
+
+export function processValueMapping(value: any, valueMapping?: DataTypeMapping): any {
+    if (valueMapping === undefined) return value;
+    const {
+        from,
+        to
+    } = valueMapping;
+    if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
+        return value.map(v => processValueMapping(v, {
+            from: valueMapping.fromSubtype!,
+            to: valueMapping.toSubtype!
+        }));
+    } else if (from === "string" && to === "number" && typeof value === "string") {
+        return Number(value);
+    } else if (from === "string" && to === "array" && valueMapping.toSubtype && typeof value === "string") {
+        return value.split(",").map((v: string) => processValueMapping(v, {
+            from: "string",
+            to: valueMapping.toSubtype!
+        }));
+    } else if (from === "string" && to === "boolean") {
+        return value === "true";
+    } else if (from === "number" && to === "boolean") {
+        return value === 1;
+    } else if (from === "boolean" && to === "number") {
+        return value ? 1 : 0;
+    } else if (from === "boolean" && to === "string") {
+        return value ? "true" : "false";
+    } else if (from === "number" && to === "string" && typeof value === "number") {
+        return value.toString();
+    } else if (from === "string" && to === "array" && typeof value === "string") {
+        return value.split(",").map((v: string) => v.trim());
+    } else if (from === "string" && to === "date" && typeof value === "string") {
+        try {
+            return new Date(value);
+        } catch (e) {
+            return value;
+        }
+    } else if (from === "date" && to === "string") {
+        return value instanceof Date && value.toISOString();
+    } else if (from === "number" && to === "date" && typeof value === "number") {
+        try {
+            return new Date(value);
+        } catch (e) {
+            return value;
+        }
+    } else if (from === "string" && to === "reference" && typeof value === "string") {
+        // split value into path and entityId (entityId is the last part of the path, after the last /)
+        const path = value.split("/").slice(0, -1).join("/");
+        const entityId = value.split("/").slice(-1)[0];
+        return new EntityReference(entityId, path);
+
+    } else if (from === to) {
+        return value;
+    } else if (from === "array" && to === "string" && Array.isArray(value)) {
+        return value.join(",");
+    }
+
+    return value;
+}

package/src/utils/file_to_json.ts

@@ -0,0 +1,83 @@
+import * as XLSX from "xlsx";
+
+export function convertFileToJson(file: File): Promise<object[]> {
+    return new Promise((resolve, reject) => {
+        // check if file is a JSON file
+        if (file.type === "application/json") {
+            const reader = new FileReader();
+            reader.onload = function (e) {
+                const data = e.target?.result as string;
+                const jsonData = JSON.parse(data);
+                resolve(jsonData);
+            };
+            reader.readAsText(file);
+        } else {
+            const reader = new FileReader();
+            reader.onload = function (e) {
+
+                const data = new Uint8Array(e.target?.result as ArrayBuffer);
+                const workbook = XLSX.read(data,
+                    {
+                        type: "array",
+                        codepage: 65001,
+                        cellDates: true,
+                    });
+                const worksheetName = workbook.SheetNames[0];
+                const worksheet = workbook.Sheets[worksheetName];
+                const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
+                const cleanedData = parsedData.map(mapJsonParse);
+                const jsonData = cleanedData.map(unflattenObject);
+                resolve(jsonData);
+            };
+            reader.readAsArrayBuffer(file);
+        }
+    });
+}
+
+function mapJsonParse(obj: Record<string, any>) {
+    return Object.keys(obj).reduce((acc: Record<string, any>, key) => {
+        try {
+            acc[key] = JSON.parse(obj[key]);
+        } catch (e) {
+            acc[key] = obj[key];
+        }
+        return acc;
+    }, {});
+}
+
+/**
+ * Take an object with keys of type `address.street`, `address.city` and
+ * convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
+ * @param flatObj
+ */
+export function unflattenObject(flatObj: { [key: string]: any }) {
+    return Object.keys(flatObj).reduce((nestedObj, key) => {
+        let currentObj = nestedObj;
+        const keyParts = key.split(".");
+        keyParts.forEach((keyPart, i) => {
+
+            if (/^[\w]+\[\d+\]$/.test(keyPart)) {
+                const mainPropertyName = keyPart.slice(0, keyPart.indexOf("["));
+                const index = parseInt(keyPart.slice(keyPart.indexOf("[") + 1, keyPart.indexOf("]")));
+
+                if (!currentObj[mainPropertyName]) {
+                    currentObj[mainPropertyName] = []
+                }
+
+                if (i !== keyParts.length - 1) {
+                    currentObj[mainPropertyName][index] = currentObj[mainPropertyName][index] || {};
+                    currentObj = currentObj[mainPropertyName][index];
+                } else {
+                    currentObj[mainPropertyName][index] = flatObj[key];
+                }
+            } else if (i !== keyParts.length - 1) {
+                currentObj[keyPart] = currentObj[keyPart] || {};
+                currentObj = currentObj[keyPart];
+            } else {
+                currentObj[keyPart] = flatObj[key];
+            }
+
+        });
+        return nestedObj;
+    }, {} as { [key: string]: any });
+}

package/src/utils/get_import_inference_type.ts

@@ -0,0 +1,27 @@
+import { DataType } from "@firecms/core";
+
+export function getInferenceType(value: any): DataType {
+    if (typeof value === "number")
+        return "number";
+    else if (typeof value === "string")
+        return "string";
+    else if (typeof value === "boolean")
+        return "boolean";
+    else if (value instanceof Date)
+        return "date";
+    else if (Array.isArray(value))
+        return "array";
+    return "map";
+}
+
+
+function isUnixTimestamp(num: number): boolean {
+    const numString = num.toString();
+    // check if the number has 13 digits
+    const isLengthValid = numString.length === 13;
+
+    // check if it falls in the expected Unix timestamp range (from 1970 to 2100)
+    const isInRange = num >= 0 && num <= 4102444800000;
+
+    return isLengthValid && isInRange;
+}

package/src/utils/get_properties_mapping.ts

@@ -0,0 +1,60 @@
+import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
+import { DataTypeMapping } from "../types";
+
+export function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping> {
+
+    function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
+
+        const dataMapping: Record<string, DataTypeMapping> = {};
+
+        Object.keys(properties).forEach((key) => {
+
+            const currentKey = namespace ? `${namespace}.${key}` : key;
+
+            const property = getPropertyInPath(properties, key) as Property;
+            const inferredProperty = getPropertyInPath(originProperties, currentKey) as Property;
+
+            if (property) {
+                if (property.dataType === "map" && property.properties) {
+                    const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
+                    Object.keys(nestedMapping).forEach((nestedKey) => {
+                        dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
+                    });
+                    return;
+                }
+
+                if (inferredProperty) {
+
+                    const from = inferredProperty.dataType;
+                    const to = property.dataType;
+                    let fromSubtype: DataType | undefined;
+                    let toSubtype: DataType | undefined;
+
+                    if (property.dataType === "array" && property.of) {
+                        toSubtype = (property.of as Property).dataType;
+                    }
+
+                    if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
+                        fromSubtype = (inferredProperty.of as Property).dataType;
+                    }
+
+                    if (from !== to || fromSubtype !== toSubtype) {
+                        console.log(`Mapping`, { currentKey, inferredProperty, property, from, to, fromSubtype, toSubtype });
+                        dataMapping[key] = {
+                            from,
+                            to,
+                            fromSubtype,
+                            toSubtype
+                        };
+                    }
+                }
+
+            }
+
+        });
+
+        return dataMapping;
+    }
+
+    return updateMapping(newProperties);
+}