@rebasepro/admin 0.0.1-canary.09e5ec5

package/src/data_import/utils/file_headers.ts

@@ -0,0 +1,90 @@
+import * as XLSX from "xlsx";
+export function getXLSXHeaders(sheet: any) {
+    let header = 0; let offset = 1;
+    const hdr = [];
+    const o:any = {};
+    if (sheet == null || sheet["!ref"] == null) return [];
+    const range = o.range !== undefined ? o.range : sheet["!ref"];
+    let r;
+    if (o.header === 1) header = 1;
+    else if (o.header === "A") header = 2;
+    else if (Array.isArray(o.header)) header = 3;
+    switch (typeof range) {
+        case "string":
+            r = safeDecodeRange(range);
+            break;
+        case "number":
+            r = safeDecodeRange(sheet["!ref"]);
+            r.s.r = range;
+            break;
+        default:
+            r = range;
+    }
+    if (header > 0) offset = 0;
+    const rr = XLSX.utils.encode_row(r.s.r);
+    const cols = new Array(r.e.c - r.s.c + 1);
+    for (let C = r.s.c; C <= r.e.c; ++C) {
+        cols[C] = XLSX.utils.encode_col(C);
+        const val = sheet[cols[C] + rr];
+        switch (header) {
+            case 1:
+                hdr.push(C);
+                break;
+            case 2:
+                hdr.push(cols[C]);
+                break;
+            case 3:
+                hdr.push(o.header[C - r.s.c]);
+                break;
+            default:
+                if (val === undefined) continue;
+                hdr.push(XLSX.utils.format_cell(val));
+        }
+    }
+    return hdr;
+}
+
+function safeDecodeRange(range:any) {
+    const o = {
+        s: {
+            c: 0,
+            r: 0
+        },
+        e: {
+            c: 0,
+            r: 0
+        }
+    };
+    let idx = 0; let i = 0; let cc = 0;
+    const len = range.length;
+    for (idx = 0; i < len; ++i) {
+        if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
+        idx = 26 * idx + cc;
+    }
+    o.s.c = --idx;
+
+    for (idx = 0; i < len; ++i) {
+        if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
+        idx = 10 * idx + cc;
+    }
+    o.s.r = --idx;
+
+    if (i === len || range.charCodeAt(++i) === 58) {
+        o.e.c = o.s.c;
+        o.e.r = o.s.r;
+        return o;
+    }
+
+    for (idx = 0; i !== len; ++i) {
+        if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
+        idx = 26 * idx + cc;
+    }
+    o.e.c = --idx;
+
+    for (idx = 0; i !== len; ++i) {
+        if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
+        idx = 10 * idx + cc;
+    }
+    o.e.r = --idx;
+    return o;
+}

package/src/data_import/utils/file_to_json.ts

@@ -0,0 +1,106 @@
+import * as XLSX from "xlsx";
+import { getXLSXHeaders } from "./file_headers";
+
+type ConversionResult = {
+    data: object[];
+    propertiesOrder: string[]
+}
+
+export function convertFileToJson(file: File): Promise<ConversionResult> {
+    return new Promise((resolve, reject) => {
+        if (file.type === "application/json") {
+            console.debug("Converting JSON file to JSON", file.name);
+            const reader = new FileReader();
+            reader.onload = function (e) {
+                try {
+                    const data = e.target?.result as string;
+                    const jsonData = JSON.parse(data);
+                    if (!Array.isArray(jsonData)) {
+                        reject(new Error("JSON file should contain an array of objects"));
+                    } else {
+                        // Assuming all objects in the array have the same structure/order
+                        const propertiesOrder = jsonData.length > 0 ? Object.keys(jsonData[0]) : [];
+                        resolve({
+                            data: jsonData,
+                            propertiesOrder
+                        });
+                    }
+                } catch (e) {
+                    console.error("Error parsing JSON file", e);
+                    reject(e);
+                }
+            };
+            reader.readAsText(file);
+        } else {
+            console.debug("Converting Excel file to JSON", file.name);
+            const reader = new FileReader();
+            reader.onload = function (e) {
+                const data = new Uint8Array(e.target?.result as ArrayBuffer);
+                const workbook = XLSX.read(data, {
+                    type: "array",
+                    codepage: 65001,
+                    cellDates: true
+                });
+                const worksheetName = workbook.SheetNames[0];
+                const worksheet = workbook.Sheets[worksheetName];
+                const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
+                const headers = getXLSXHeaders(worksheet);
+                const cleanedData = parsedData.map(mapJsonParse);
+                const jsonData = cleanedData.map(unflattenObject);
+                resolve({
+                    data: jsonData,
+                    propertiesOrder: headers
+                });
+            };
+            reader.readAsArrayBuffer(file);
+        }
+    });
+}
+
+function mapJsonParse(obj: Record<string, any>) {
+    return Object.keys(obj).reduce((acc: Record<string, any>, key) => {
+        try {
+            acc[key] = JSON.parse(obj[key]);
+        } catch (e) {
+            acc[key] = obj[key];
+        }
+        return acc;
+    }, {});
+}
+
+/**
+ * Take an object with keys of type `address.street`, `address.city` and
+ * convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
+ * @param flatObj
+ */
+export function unflattenObject(flatObj: { [key: string]: any }) {
+    return Object.keys(flatObj).reduce((nestedObj, key) => {
+        let currentObj = nestedObj;
+        const keyParts = key.split(".");
+        keyParts.forEach((keyPart, i) => {
+
+            if (/^[\w]+\[\d+\]$/.test(keyPart)) {
+                const mainPropertyName = keyPart.slice(0, keyPart.indexOf("["));
+                const index = parseInt(keyPart.slice(keyPart.indexOf("[") + 1, keyPart.indexOf("]")));
+
+                if (!currentObj[mainPropertyName]) {
+                    currentObj[mainPropertyName] = []
+                }
+
+                if (i !== keyParts.length - 1) {
+                    currentObj[mainPropertyName][index] = currentObj[mainPropertyName][index] || {};
+                    currentObj = currentObj[mainPropertyName][index];
+                } else {
+                    currentObj[mainPropertyName][index] = flatObj[key];
+                }
+            } else if (i !== keyParts.length - 1) {
+                currentObj[keyPart] = currentObj[keyPart] || {};
+                currentObj = currentObj[keyPart];
+            } else {
+                currentObj[keyPart] = flatObj[key];
+            }
+
+        });
+        return nestedObj;
+    }, {} as { [key: string]: any });
+}

package/src/data_import/utils/get_import_inference_type.ts

@@ -0,0 +1,27 @@
+import { DataType } from "@rebasepro/types";
+
+export function getInferenceType(value: any): DataType {
+    if (typeof value === "number")
+        return "number";
+    else if (typeof value === "string")
+        return "string";
+    else if (typeof value === "boolean")
+        return "boolean";
+    else if (value instanceof Date)
+        return "date";
+    else if (Array.isArray(value))
+        return "array";
+    return "map";
+}
+
+
+function isUnixTimestamp(num: number): boolean {
+    const numString = num.toString();
+    // check if the number has 13 digits
+    const isLengthValid = numString.length === 13;
+
+    // check if it falls in the expected Unix timestamp range (from 1970 to 2100)
+    const isInRange = num >= 0 && num <= 4102444800000;
+
+    return isLengthValid && isInRange;
+}

package/src/data_import/utils/get_properties_mapping.ts

@@ -0,0 +1,67 @@
+// import {
+//     type,
+//     getPropertyInPath
+// } from "@rebasepro/core";
+import { Properties, Property } from "@rebasepro/types";
+// import { typeMapping } from "@rebasepro/types";
+//
+// export function getPropertiesMapping(originProperties: Properties,
+//                                      newProperties: Properties,
+//                                      headersMapping: Record<string, string | null>): Record<string, typeMapping> {
+//
+//     function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, typeMapping> {
+//
+//         const dataMapping: Record<string, typeMapping> = {};
+//
+//         Object.keys(properties).forEach((key) => {
+//
+//             const currentKey = namespace ? `${namespace}.${key}` : key;
+//
+//             const property = getPropertyInPath(properties, key) as Property;
+//             // reverse lookup
+//             const mappedKey = Object.entries(headersMapping).find(([_, value]) => value === currentKey)?.[0];
+//             const inferredProperty = mappedKey ? getPropertyInPath(originProperties, mappedKey) as Property : null;
+//
+//             if (property) {
+//                 if (property.type === "map" && property.properties) {
+//                     const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
+//                     Object.keys(nestedMapping).forEach((nestedKey) => {
+//                         dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
+//                     });
+//                     return;
+//                 }
+//
+//                 if (inferredProperty) {
+//
+//                     const from = inferredProperty.type;
+//                     const to = property.type;
+//                     let fromSubtype: type | undefined;
+//                     let toSubtype: type | undefined;
+//
+//                     if (property.type === "array" && property.of) {
+//                         toSubtype = (property.of as Property).type;
+//                     }
+//
+//                     if (inferredProperty?.type === "array" && inferredProperty?.of) {
+//                         fromSubtype = (inferredProperty.of as Property).type;
+//                     }
+//
+//                     if (from !== to || fromSubtype !== toSubtype) {
+//                         dataMapping[currentKey] = {
+//                             from,
+//                             to,
+//                             fromSubtype,
+//                             toSubtype
+//                         };
+//                     }
+//                 }
+//
+//             }
+//
+//         });
+//
+//         return dataMapping;
+//     }
+//
+//     return updateMapping(newProperties);
+// }

package/src/data_import/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./file_to_json";
+export * from "./data";
+export * from "./get_import_inference_type";