@firecms/data_import_export 3.0.0-canary.19 → 3.0.0-canary.191

package/src/types/column_mapping.ts

@@ -1,32 +0,0 @@
-import React from "react";
-import { DataType, Entity, Property } from "@firecms/core";
-
-export type ImportConfig = {
-
-    inUse: boolean;
-    setInUse: React.Dispatch<React.SetStateAction<boolean>>;
-
-    idColumn: string | undefined;
-    setIdColumn: React.Dispatch<React.SetStateAction<string | undefined>>;
-
-    importData: object[];
-    setImportData: React.Dispatch<React.SetStateAction<object[]>>;
-
-    entities: Entity<any>[];
-    setEntities: React.Dispatch<React.SetStateAction<Entity<any>[]>>;
-
-    // mapping of the column name in the import file to the property key in the data model
-    headersMapping: Record<string, string | null>;
-    setHeadersMapping: React.Dispatch<React.SetStateAction<Record<string, string | null>>>;
-
-    originProperties: Record<string, Property>;
-    setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
-
-}
-
-export type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
-}

package/src/types/index.ts

@@ -1 +0,0 @@
-export * from "./column_mapping";

package/src/utils/data.ts

@@ -1,223 +0,0 @@
-import { DataType, Entity, EntityReference, getPropertyInPath, Properties } from "@firecms/core";
-import { unflattenObject } from "./file_to_json";
-
-type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
-}
-
-export function convertDataToEntity(data: Record<any, any>,
-                                    idColumn: string | undefined,
-                                    headersMapping: Record<string, string | null>,
-                                    properties: Properties,
-                                    propertiesMapping: Record<string, DataTypeMapping>,
-                                    path: string): Entity<any> {
-    const flatObject = flattenEntry(data);
-    if (idColumn)
-        delete flatObject[idColumn];
-    const mappedKeysObject = Object.entries(flatObject)
-        .map(([key, value]) => {
-            const mappedKey = headersMapping[key] ?? key;
-
-            const mappedProperty = getPropertyInPath(properties, mappedKey);
-            if (!mappedProperty)
-                return {};
-
-            const propertyMapping = propertiesMapping[mappedKey];
-            let valueResult = value;
-            if (propertyMapping) {
-                valueResult = processValueMapping(value, propertyMapping);
-            }
-            return ({
-                [mappedKey]: valueResult
-            });
-        })
-        .reduce((acc, curr) => ({ ...acc, ...curr }), {});
-    const values = unflattenObject(mappedKeysObject);
-    let id = idColumn ? data[idColumn] : undefined;
-    if (typeof id === "string") {
-        id = id.trim();
-    } else if (typeof id === "number") {
-        id = id.toString();
-    } else if (typeof id === "boolean") {
-        id = id.toString();
-    } else if (id instanceof Date) {
-        id = id.toISOString();
-    } else if (id && "toString" in id) {
-        id = id.toString();
-    }
-
-    return {
-        id,
-        values,
-        path
-    };
-}
-
-export function flattenEntry(obj: any, parent = ""): any {
-    return Object.keys(obj).reduce((acc, key) => {
-        const prefixedKey = parent ? `${parent}.${key}` : key;
-
-        if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
-            Object.assign(acc, flattenEntry(obj[key], prefixedKey));
-        } else {
-            // @ts-ignore
-            acc[prefixedKey] = obj[key];
-        }
-
-        return acc;
-    }, {});
-}
-
-// export function convertDataEntryValue({
-//                                           key,
-//                                           fullKey,
-//                                           value,
-//                                           idColumn,
-//                                           headersMapping,
-//                                           properties,
-//                                           propertiesMapping
-//                                       }: {
-//     key?: string,
-//     fullKey: string,
-//     value: any,
-//     idColumn?: string,
-//     headersMapping: Record<string, string | null>,
-//     properties: Properties,
-//     propertiesMapping: Record<string, DataTypeMapping>
-// }): any {
-//
-//     if (value === undefined) return value;
-//     if (value === null) return value;
-//
-//     if (Array.isArray(value)) {
-//         const mappedKey = headersMapping[fullKey] || fullKey;
-//         const valueMapping = propertiesMapping[mappedKey];
-//         return processValueMapping(value, valueMapping);
-//         // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
-//     } else if (typeof value === "object") {
-//         return convertDataObjectValue({
-//             key,
-//             fullKey,
-//             value,
-//             idColumn,
-//             headersMapping,
-//             properties,
-//             propertiesMapping
-//         });
-//     } else {
-//         const mappedKey = headersMapping[fullKey] || fullKey;
-//         const valueMapping = propertiesMapping[mappedKey];
-//         return processValueMapping(value, valueMapping);
-//     }
-// }
-
-// export function convertDataObjectValue({
-//                                            key,
-//                                            fullKey,
-//                                            value,
-//                                            idColumn,
-//                                            headersMapping,
-//                                            properties,
-//                                            propertiesMapping
-//                                        }: {
-//     key?: string,
-//     fullKey?: string,
-//     value: object,
-//     idColumn?: string,
-//     headersMapping: Record<string, string | null>,
-//     properties: Properties,
-//     propertiesMapping: Record<string, DataTypeMapping>
-// }): object {
-//
-//     if (value instanceof Date) {
-//         const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
-//         const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
-//         return processValueMapping(value, valueMapping);
-//     }
-//
-//     return Object.entries(value)
-//         .map(([childKey, childValue]) => {
-//             const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
-//             const mappedKey = headersMapping[childFullKey] ?? childFullKey;
-//             const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
-//             const property = getPropertyInPath(properties, mappedKey);
-//             // if (!property) {
-//             //     console.log("ppp", { properties, mappedKey,  })
-//             //     return {};
-//             // }
-//             return ({
-//                 [mappedKeyLastPart]: convertDataEntryValue({
-//                     key: childKey,
-//                     fullKey: childFullKey,
-//                     value: childValue,
-//                     idColumn,
-//                     headersMapping,
-//                     properties,
-//                     propertiesMapping
-//                 })
-//             });
-//         })
-//         .reduce((acc, curr) => ({ ...acc, ...curr }), {});
-// }
-
-export function processValueMapping(value: any, valueMapping?: DataTypeMapping): any {
-    if (valueMapping === undefined) return value;
-    const {
-        from,
-        to
-    } = valueMapping;
-    if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
-        return value.map(v => processValueMapping(v, {
-            from: valueMapping.fromSubtype!,
-            to: valueMapping.toSubtype!
-        }));
-    } else if (from === "string" && to === "number" && typeof value === "string") {
-        return Number(value);
-    } else if (from === "string" && to === "array" && valueMapping.toSubtype && typeof value === "string") {
-        return value.split(",").map((v: string) => processValueMapping(v, {
-            from: "string",
-            to: valueMapping.toSubtype!
-        }));
-    } else if (from === "string" && to === "boolean") {
-        return value === "true";
-    } else if (from === "number" && to === "boolean") {
-        return value === 1;
-    } else if (from === "boolean" && to === "number") {
-        return value ? 1 : 0;
-    } else if (from === "boolean" && to === "string") {
-        return value ? "true" : "false";
-    } else if (from === "number" && to === "string" && typeof value === "number") {
-        return value.toString();
-    } else if (from === "string" && to === "array" && typeof value === "string") {
-        return value.split(",").map((v: string) => v.trim());
-    } else if (from === "string" && to === "date" && typeof value === "string") {
-        try {
-            return new Date(value);
-        } catch (e) {
-            return value;
-        }
-    } else if (from === "date" && to === "string") {
-        return value instanceof Date && value.toISOString();
-    } else if (from === "number" && to === "date" && typeof value === "number") {
-        try {
-            return new Date(value);
-        } catch (e) {
-            return value;
-        }
-    } else if (from === "string" && to === "reference" && typeof value === "string") {
-        // split value into path and entityId (entityId is the last part of the path, after the last /)
-        const path = value.split("/").slice(0, -1).join("/");
-        const entityId = value.split("/").slice(-1)[0];
-        return new EntityReference(entityId, path);
-
-    } else if (from === to) {
-        return value;
-    } else if (from === "array" && to === "string" && Array.isArray(value)) {
-        return value.join(",");
-    }
-
-    return value;
-}

package/src/utils/file_to_json.ts

@@ -1,88 +0,0 @@
-import * as XLSX from "xlsx";
-
-export function convertFileToJson(file: File): Promise<object[]> {
-    return new Promise((resolve, reject) => {
-        // check if file is a JSON file
-        if (file.type === "application/json") {
-            console.debug("Converting JSON file to JSON", file.name);
-            const reader = new FileReader();
-            reader.onload = function (e) {
-                const data = e.target?.result as string;
-                const jsonData = JSON.parse(data);
-                if (!Array.isArray(jsonData)) {
-                    reject(new Error("JSON file should contain an array of objects"));
-                }
-                resolve(jsonData);
-            };
-            reader.readAsText(file);
-        } else {
-            console.debug("Converting Excel file to JSON", file.name);
-            const reader = new FileReader();
-            reader.onload = function (e) {
-
-                const data = new Uint8Array(e.target?.result as ArrayBuffer);
-                const workbook = XLSX.read(data,
-                    {
-                        type: "array",
-                        codepage: 65001,
-                        cellDates: true,
-                    });
-                const worksheetName = workbook.SheetNames[0];
-                const worksheet = workbook.Sheets[worksheetName];
-                const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
-                const cleanedData = parsedData.map(mapJsonParse);
-                const jsonData = cleanedData.map(unflattenObject);
-                resolve(jsonData);
-            };
-            reader.readAsArrayBuffer(file);
-        }
-    });
-}
-
-function mapJsonParse(obj: Record<string, any>) {
-    return Object.keys(obj).reduce((acc: Record<string, any>, key) => {
-        try {
-            acc[key] = JSON.parse(obj[key]);
-        } catch (e) {
-            acc[key] = obj[key];
-        }
-        return acc;
-    }, {});
-}
-
-/**
- * Take an object with keys of type `address.street`, `address.city` and
- * convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
- * @param flatObj
- */
-export function unflattenObject(flatObj: { [key: string]: any }) {
-    return Object.keys(flatObj).reduce((nestedObj, key) => {
-        let currentObj = nestedObj;
-        const keyParts = key.split(".");
-        keyParts.forEach((keyPart, i) => {
-
-            if (/^[\w]+\[\d+\]$/.test(keyPart)) {
-                const mainPropertyName = keyPart.slice(0, keyPart.indexOf("["));
-                const index = parseInt(keyPart.slice(keyPart.indexOf("[") + 1, keyPart.indexOf("]")));
-
-                if (!currentObj[mainPropertyName]) {
-                    currentObj[mainPropertyName] = []
-                }
-
-                if (i !== keyParts.length - 1) {
-                    currentObj[mainPropertyName][index] = currentObj[mainPropertyName][index] || {};
-                    currentObj = currentObj[mainPropertyName][index];
-                } else {
-                    currentObj[mainPropertyName][index] = flatObj[key];
-                }
-            } else if (i !== keyParts.length - 1) {
-                currentObj[keyPart] = currentObj[keyPart] || {};
-                currentObj = currentObj[keyPart];
-            } else {
-                currentObj[keyPart] = flatObj[key];
-            }
-
-        });
-        return nestedObj;
-    }, {} as { [key: string]: any });
-}

package/src/utils/get_import_inference_type.ts

@@ -1,27 +0,0 @@
-import { DataType } from "@firecms/core";
-
-export function getInferenceType(value: any): DataType {
-    if (typeof value === "number")
-        return "number";
-    else if (typeof value === "string")
-        return "string";
-    else if (typeof value === "boolean")
-        return "boolean";
-    else if (value instanceof Date)
-        return "date";
-    else if (Array.isArray(value))
-        return "array";
-    return "map";
-}
-
-
-function isUnixTimestamp(num: number): boolean {
-    const numString = num.toString();
-    // check if the number has 13 digits
-    const isLengthValid = numString.length === 13;
-
-    // check if it falls in the expected Unix timestamp range (from 1970 to 2100)
-    const isInRange = num >= 0 && num <= 4102444800000;
-
-    return isLengthValid && isInRange;
-}

package/src/utils/get_properties_mapping.ts

@@ -1,59 +0,0 @@
-import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
-import { DataTypeMapping } from "../types";
-
-export function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping> {
-
-    function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
-
-        const dataMapping: Record<string, DataTypeMapping> = {};
-
-        Object.keys(properties).forEach((key) => {
-
-            const currentKey = namespace ? `${namespace}.${key}` : key;
-
-            const property = getPropertyInPath(properties, key) as Property;
-            const inferredProperty = getPropertyInPath(originProperties, currentKey) as Property;
-
-            if (property) {
-                if (property.dataType === "map" && property.properties) {
-                    const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
-                    Object.keys(nestedMapping).forEach((nestedKey) => {
-                        dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
-                    });
-                    return;
-                }
-
-                if (inferredProperty) {
-
-                    const from = inferredProperty.dataType;
-                    const to = property.dataType;
-                    let fromSubtype: DataType | undefined;
-                    let toSubtype: DataType | undefined;
-
-                    if (property.dataType === "array" && property.of) {
-                        toSubtype = (property.of as Property).dataType;
-                    }
-
-                    if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
-                        fromSubtype = (inferredProperty.of as Property).dataType;
-                    }
-
-                    if (from !== to || fromSubtype !== toSubtype) {
-                        dataMapping[key] = {
-                            from,
-                            to,
-                            fromSubtype,
-                            toSubtype
-                        };
-                    }
-                }
-
-            }
-
-        });
-
-        return dataMapping;
-    }
-
-    return updateMapping(newProperties);
-}

package/src/utils/index.ts

@@ -1,4 +0,0 @@
-export * from "./file_to_json";
-export * from "./data";
-export * from "./get_import_inference_type";
-export * from "./get_properties_mapping";