@firecms/data_import_export 3.0.0-canary.5 → 3.0.0-canary.51

package/src/utils/data.ts

@@ -1,43 +1,59 @@
-import { DataType, Entity, EntityReference, getPropertyInPath, Properties } from "@firecms/core";
+import {
+    Entity,
+    EntityReference,
+    getPropertyInPath,
+    isPropertyBuilder,
+    mergeDeep,
+    Properties,
+    Property,
+    PropertyOrBuilder,
+    ResolvedProperty,
+    resolveProperty
+} from "@firecms/core";
 import { unflattenObject } from "./file_to_json";
-
-type DataTypeMapping = {
-    from: DataType;
-    fromSubtype?: DataType;
-    to: DataType;
-    toSubtype?: DataType;
-}
+import { getIn } from "@firecms/formex";
+import { inferTypeFromValue } from "@firecms/schema_inference";
 
 export function convertDataToEntity(data: Record<any, any>,
                                     idColumn: string | undefined,
                                     headersMapping: Record<string, string | null>,
                                     properties: Properties,
-                                    propertiesMapping: Record<string, DataTypeMapping>,
-                                    path: string): Entity<any> {
+                                    path: string,
+                                    defaultValues: Record<string, any>): Entity<any> {
     const flatObject = flattenEntry(data);
     if (idColumn)
         delete flatObject[idColumn];
     const mappedKeysObject = Object.entries(flatObject)
         .map(([key, value]) => {
-            const mappedKey = headersMapping[key] ?? key;
+            const mappedKey = getIn(headersMapping, key) ?? key;
 
             const mappedProperty = getPropertyInPath(properties, mappedKey);
-            if (!mappedProperty)
+            if (!mappedProperty) {
                 return {};
-
-            const propertyMapping = propertiesMapping[mappedKey];
-            let valueResult = value;
-            if (propertyMapping) {
-                valueResult = processValueMapping(value, propertyMapping);
             }
+            const processedValue = processValueMapping(value, mappedProperty);
             return ({
-                [mappedKey]: valueResult
+                [mappedKey]: processedValue
             });
         })
         .reduce((acc, curr) => ({ ...acc, ...curr }), {});
-    const values = unflattenObject(mappedKeysObject);
+
+    const values = mergeDeep(defaultValues ?? {}, unflattenObject(mappedKeysObject));
+    let id = idColumn ? data[idColumn] : undefined;
+    if (typeof id === "string") {
+        id = id.trim();
+    } else if (typeof id === "number") {
+        id = id.toString();
+    } else if (typeof id === "boolean") {
+        id = id.toString();
+    } else if (id instanceof Date) {
+        id = id.toISOString();
+    } else if (id && "toString" in id) {
+        id = id.toString();
+    }
+
     return {
-        id: idColumn ? data[idColumn] : undefined,
+        id,
         values,
         path
     };
@@ -47,7 +63,7 @@ export function flattenEntry(obj: any, parent = ""): any {
     return Object.keys(obj).reduce((acc, key) => {
         const prefixedKey = parent ? `${parent}.${key}` : key;
 
-        if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
+        if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
             Object.assign(acc, flattenEntry(obj[key], prefixedKey));
         } else {
             // @ts-ignore
@@ -58,116 +74,23 @@ export function flattenEntry(obj: any, parent = ""): any {
     }, {});
 }
 
-// export function convertDataEntryValue({
-//                                           key,
-//                                           fullKey,
-//                                           value,
-//                                           idColumn,
-//                                           headersMapping,
-//                                           properties,
-//                                           propertiesMapping
-//                                       }: {
-//     key?: string,
-//     fullKey: string,
-//     value: any,
-//     idColumn?: string,
-//     headersMapping: Record<string, string | null>,
-//     properties: Properties,
-//     propertiesMapping: Record<string, DataTypeMapping>
-// }): any {
-//
-//     if (value === undefined) return value;
-//     if (value === null) return value;
-//
-//     if (Array.isArray(value)) {
-//         const mappedKey = headersMapping[fullKey] || fullKey;
-//         const valueMapping = propertiesMapping[mappedKey];
-//         return processValueMapping(value, valueMapping);
-//         // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
-//     } else if (typeof value === "object") {
-//         return convertDataObjectValue({
-//             key,
-//             fullKey,
-//             value,
-//             idColumn,
-//             headersMapping,
-//             properties,
-//             propertiesMapping
-//         });
-//     } else {
-//         const mappedKey = headersMapping[fullKey] || fullKey;
-//         const valueMapping = propertiesMapping[mappedKey];
-//         return processValueMapping(value, valueMapping);
-//     }
-// }
+export function processValueMapping(value: any, property?: PropertyOrBuilder): any {
+    if (value === null) return null;
 
-// export function convertDataObjectValue({
-//                                            key,
-//                                            fullKey,
-//                                            value,
-//                                            idColumn,
-//                                            headersMapping,
-//                                            properties,
-//                                            propertiesMapping
-//                                        }: {
-//     key?: string,
-//     fullKey?: string,
-//     value: object,
-//     idColumn?: string,
-//     headersMapping: Record<string, string | null>,
-//     properties: Properties,
-//     propertiesMapping: Record<string, DataTypeMapping>
-// }): object {
-//
-//     if (value instanceof Date) {
-//         const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
-//         const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
-//         return processValueMapping(value, valueMapping);
-//     }
-//
-//     return Object.entries(value)
-//         .map(([childKey, childValue]) => {
-//             const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
-//             const mappedKey = headersMapping[childFullKey] ?? childFullKey;
-//             const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
-//             const property = getPropertyInPath(properties, mappedKey);
-//             // if (!property) {
-//             //     console.log("ppp", { properties, mappedKey,  })
-//             //     return {};
-//             // }
-//             return ({
-//                 [mappedKeyLastPart]: convertDataEntryValue({
-//                     key: childKey,
-//                     fullKey: childFullKey,
-//                     value: childValue,
-//                     idColumn,
-//                     headersMapping,
-//                     properties,
-//                     propertiesMapping
-//                 })
-//             });
-//         })
-//         .reduce((acc, curr) => ({ ...acc, ...curr }), {});
-// }
+    if (property === undefined) return value;
+    const usedProperty: ResolvedProperty | null = resolveProperty({
+        propertyOrBuilder: property,
+    })
+    if (usedProperty === null) return value;
+    const from = inferTypeFromValue(value);
+    const to = usedProperty.dataType;
 
-export function processValueMapping(value: any, valueMapping?: DataTypeMapping): any {
-    if (valueMapping === undefined) return value;
-    const {
-        from,
-        to
-    } = valueMapping;
-    if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
-        return value.map(v => processValueMapping(v, {
-            from: valueMapping.fromSubtype!,
-            to: valueMapping.toSubtype!
-        }));
+    if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
+        return value.map(v => processValueMapping(v, usedProperty.of as Property));
     } else if (from === "string" && to === "number" && typeof value === "string") {
         return Number(value);
-    } else if (from === "string" && to === "array" && valueMapping.toSubtype && typeof value === "string") {
-        return value.split(",").map((v: string) => processValueMapping(v, {
-            from: "string",
-            to: valueMapping.toSubtype!
-        }));
+    } else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
+        return value.split(",").map((v: string) => processValueMapping(v, usedProperty.of));
     } else if (from === "string" && to === "boolean") {
         return value === "true";
     } else if (from === "number" && to === "boolean") {

package/src/utils/file_headers.ts

@@ -0,0 +1,90 @@
+import * as XLSX from "xlsx";
+export function getXLSXHeaders(sheet: any) {
+    let header = 0; let offset = 1;
+    const hdr = [];
+    const o:any = {};
+    if (sheet == null || sheet["!ref"] == null) return [];
+    const range = o.range !== undefined ? o.range : sheet["!ref"];
+    let r;
+    if (o.header === 1) header = 1;
+    else if (o.header === "A") header = 2;
+    else if (Array.isArray(o.header)) header = 3;
+    switch (typeof range) {
+        case "string":
+            r = safeDecodeRange(range);
+            break;
+        case "number":
+            r = safeDecodeRange(sheet["!ref"]);
+            r.s.r = range;
+            break;
+        default:
+            r = range;
+    }
+    if (header > 0) offset = 0;
+    const rr = XLSX.utils.encode_row(r.s.r);
+    const cols = new Array(r.e.c - r.s.c + 1);
+    for (let C = r.s.c; C <= r.e.c; ++C) {
+        cols[C] = XLSX.utils.encode_col(C);
+        const val = sheet[cols[C] + rr];
+        switch (header) {
+            case 1:
+                hdr.push(C);
+                break;
+            case 2:
+                hdr.push(cols[C]);
+                break;
+            case 3:
+                hdr.push(o.header[C - r.s.c]);
+                break;
+            default:
+                if (val === undefined) continue;
+                hdr.push(XLSX.utils.format_cell(val));
+        }
+    }
+    return hdr;
+}
+
+function safeDecodeRange(range:any) {
+    const o = {
+        s: {
+            c: 0,
+            r: 0
+        },
+        e: {
+            c: 0,
+            r: 0
+        }
+    };
+    let idx = 0; let i = 0; let cc = 0;
+    const len = range.length;
+    for (idx = 0; i < len; ++i) {
+        if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
+        idx = 26 * idx + cc;
+    }
+    o.s.c = --idx;
+
+    for (idx = 0; i < len; ++i) {
+        if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
+        idx = 10 * idx + cc;
+    }
+    o.s.r = --idx;
+
+    if (i === len || range.charCodeAt(++i) === 58) {
+        o.e.c = o.s.c;
+        o.e.r = o.s.r;
+        return o;
+    }
+
+    for (idx = 0; i !== len; ++i) {
+        if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
+        idx = 26 * idx + cc;
+    }
+    o.e.c = --idx;
+
+    for (idx = 0; i !== len; ++i) {
+        if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
+        idx = 10 * idx + cc;
+    }
+    o.e.r = --idx;
+    return o;
+}

package/src/utils/file_to_json.ts

@@ -1,38 +1,56 @@
 import * as XLSX from "xlsx";
+import { getXLSXHeaders } from "./file_headers";
 
-export function convertFileToJson(file: File): Promise<object[]> {
+type ConversionResult = {
+    data: object[];
+    propertiesOrder: string[]
+}
+
+export function convertFileToJson(file: File): Promise<ConversionResult> {
     return new Promise((resolve, reject) => {
-        // check if file is a JSON file
         if (file.type === "application/json") {
             console.debug("Converting JSON file to JSON", file.name);
             const reader = new FileReader();
             reader.onload = function (e) {
-                const data = e.target?.result as string;
-                const jsonData = JSON.parse(data);
-                if (!Array.isArray(jsonData)) {
-                    reject(new Error("JSON file should contain an array of objects"));
+                try {
+                    const data = e.target?.result as string;
+                    const jsonData = JSON.parse(data);
+                    if (!Array.isArray(jsonData)) {
+                        reject(new Error("JSON file should contain an array of objects"));
+                    } else {
+                        // Assuming all objects in the array have the same structure/order
+                        const propertiesOrder = jsonData.length > 0 ? Object.keys(jsonData[0]) : [];
+                        resolve({
+                            data: jsonData,
+                            propertiesOrder
+                        });
+                    }
+                } catch (e) {
+                    console.error("Error parsing JSON file", e);
+                    reject(e);
                 }
-                resolve(jsonData);
             };
             reader.readAsText(file);
         } else {
             console.debug("Converting Excel file to JSON", file.name);
             const reader = new FileReader();
             reader.onload = function (e) {
-
                 const data = new Uint8Array(e.target?.result as ArrayBuffer);
-                const workbook = XLSX.read(data,
-                    {
-                        type: "array",
-                        codepage: 65001,
-                        cellDates: true,
-                    });
+                const workbook = XLSX.read(data, {
+                    type: "array",
+                    codepage: 65001,
+                    cellDates: true,
+                });
                 const worksheetName = workbook.SheetNames[0];
                 const worksheet = workbook.Sheets[worksheetName];
                 const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
+                const headers = getXLSXHeaders(worksheet);
                 const cleanedData = parsedData.map(mapJsonParse);
                 const jsonData = cleanedData.map(unflattenObject);
-                resolve(jsonData);
+                resolve({
+                    data: jsonData,
+                    propertiesOrder: headers
+                });
             };
             reader.readAsArrayBuffer(file);
         }

package/src/utils/get_properties_mapping.ts

@@ -1,59 +1,63 @@
-import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
-import { DataTypeMapping } from "../types";
-
-export function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping> {
-
-    function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
-
-        const dataMapping: Record<string, DataTypeMapping> = {};
-
-        Object.keys(properties).forEach((key) => {
-
-            const currentKey = namespace ? `${namespace}.${key}` : key;
-
-            const property = getPropertyInPath(properties, key) as Property;
-            const inferredProperty = getPropertyInPath(originProperties, currentKey) as Property;
-
-            if (property) {
-                if (property.dataType === "map" && property.properties) {
-                    const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
-                    Object.keys(nestedMapping).forEach((nestedKey) => {
-                        dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
-                    });
-                    return;
-                }
-
-                if (inferredProperty) {
-
-                    const from = inferredProperty.dataType;
-                    const to = property.dataType;
-                    let fromSubtype: DataType | undefined;
-                    let toSubtype: DataType | undefined;
-
-                    if (property.dataType === "array" && property.of) {
-                        toSubtype = (property.of as Property).dataType;
-                    }
-
-                    if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
-                        fromSubtype = (inferredProperty.of as Property).dataType;
-                    }
-
-                    if (from !== to || fromSubtype !== toSubtype) {
-                        dataMapping[key] = {
-                            from,
-                            to,
-                            fromSubtype,
-                            toSubtype
-                        };
-                    }
-                }
-
-            }
-
-        });
-
-        return dataMapping;
-    }
-
-    return updateMapping(newProperties);
-}
+// import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
+// import { DataTypeMapping } from "../types";
+//
+// export function getPropertiesMapping(originProperties: Properties,
+//                                      newProperties: Properties,
+//                                      headersMapping: Record<string, string | null>): Record<string, DataTypeMapping> {
+//
+//     function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
+//
+//         const dataMapping: Record<string, DataTypeMapping> = {};
+//
+//         Object.keys(properties).forEach((key) => {
+//
+//             const currentKey = namespace ? `${namespace}.${key}` : key;
+//
+//             const property = getPropertyInPath(properties, key) as Property;
+//             // reverse lookup
+//             const mappedKey = Object.entries(headersMapping).find(([_, value]) => value === currentKey)?.[0];
+//             const inferredProperty = mappedKey ? getPropertyInPath(originProperties, mappedKey) as Property : null;
+//
+//             if (property) {
+//                 if (property.dataType === "map" && property.properties) {
+//                     const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
+//                     Object.keys(nestedMapping).forEach((nestedKey) => {
+//                         dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
+//                     });
+//                     return;
+//                 }
+//
+//                 if (inferredProperty) {
+//
+//                     const from = inferredProperty.dataType;
+//                     const to = property.dataType;
+//                     let fromSubtype: DataType | undefined;
+//                     let toSubtype: DataType | undefined;
+//
+//                     if (property.dataType === "array" && property.of) {
+//                         toSubtype = (property.of as Property).dataType;
+//                     }
+//
+//                     if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
+//                         fromSubtype = (inferredProperty.of as Property).dataType;
+//                     }
+//
+//                     if (from !== to || fromSubtype !== toSubtype) {
+//                         dataMapping[currentKey] = {
+//                             from,
+//                             to,
+//                             fromSubtype,
+//                             toSubtype
+//                         };
+//                     }
+//                 }
+//
+//             }
+//
+//         });
+//
+//         return dataMapping;
+//     }
+//
+//     return updateMapping(newProperties);
+// }

package/src/utils/index.ts

@@ -1,4 +1,3 @@
 export * from "./file_to_json";
 export * from "./data";
 export * from "./get_import_inference_type";
-export * from "./get_properties_mapping";