@firecms/data_import_export 3.0.0-3.0.0-beta.4.pre.1.0

package/dist/types/column_mapping.d.ts

@@ -0,0 +1,22 @@
+import React from "react";
+import { DataType, Entity, Property } from "@firecms/core";
+export type ImportConfig = {
+    inUse: boolean;
+    setInUse: React.Dispatch<React.SetStateAction<boolean>>;
+    idColumn: string | undefined;
+    setIdColumn: React.Dispatch<React.SetStateAction<string | undefined>>;
+    importData: object[];
+    setImportData: React.Dispatch<React.SetStateAction<object[]>>;
+    entities: Entity<any>[];
+    setEntities: React.Dispatch<React.SetStateAction<Entity<any>[]>>;
+    headersMapping: Record<string, string | null>;
+    setHeadersMapping: React.Dispatch<React.SetStateAction<Record<string, string | null>>>;
+    originProperties: Record<string, Property>;
+    setOriginProperties: React.Dispatch<React.SetStateAction<Record<string, Property>>>;
+};
+export type DataTypeMapping = {
+    from: DataType;
+    fromSubtype?: DataType;
+    to: DataType;
+    toSubtype?: DataType;
+};

package/dist/types/index.d.ts

@@ -0,0 +1 @@
+export * from "./column_mapping";

package/dist/useImportExportPlugin.d.ts

@@ -0,0 +1,16 @@
+import React from "react";
+import { EntityCollection, FireCMSPlugin } from "@firecms/core";
+/**
+ *
+ */
+export declare function useImportExportPlugin(props?: ImportExportPluginProps): FireCMSPlugin<any, any, any, ImportExportPluginProps>;
+export type ImportExportPluginProps = {
+    exportAllowed?: (props: ExportAllowedParams) => boolean;
+    notAllowedView: React.ReactNode;
+    onAnalyticsEvent?: (event: string, params?: any) => void;
+};
+export type ExportAllowedParams = {
+    collectionEntitiesCount: number;
+    path: string;
+    collection: EntityCollection;
+};

package/dist/utils/data.d.ts

@@ -0,0 +1,11 @@
+import { DataType, Entity, Properties } from "@firecms/core";
+type DataTypeMapping = {
+    from: DataType;
+    fromSubtype?: DataType;
+    to: DataType;
+    toSubtype?: DataType;
+};
+export declare function convertDataToEntity(data: Record<any, any>, idColumn: string | undefined, headersMapping: Record<string, string | null>, properties: Properties, propertiesMapping: Record<string, DataTypeMapping>, path: string): Entity<any>;
+export declare function flattenEntry(obj: any, parent?: string): any;
+export declare function processValueMapping(value: any, valueMapping?: DataTypeMapping): any;
+export {};

package/dist/utils/file_to_json.d.ts

@@ -0,0 +1,11 @@
+export declare function convertFileToJson(file: File): Promise<object[]>;
+/**
+ * Take an object with keys of type `address.street`, `address.city` and
+ * convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
+ * @param flatObj
+ */
+export declare function unflattenObject(flatObj: {
+    [key: string]: any;
+}): {
+    [key: string]: any;
+};

package/dist/utils/get_import_inference_type.d.ts

@@ -0,0 +1,2 @@
+import { DataType } from "@firecms/core";
+export declare function getInferenceType(value: any): DataType;

package/dist/utils/get_properties_mapping.d.ts

@@ -0,0 +1,3 @@
+import { Properties } from "@firecms/core";
+import { DataTypeMapping } from "../types";
+export declare function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping>;

package/dist/utils/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./file_to_json";
+export * from "./data";
+export * from "./get_import_inference_type";
+export * from "./get_properties_mapping";

package/package.json

@@ -0,0 +1,103 @@
+{
+  "name": "@firecms/data_import_export",
+  "type": "module",
+  "version": "3.0.0-3.0.0-beta.4.pre.1.0",
+  "access": "public",
+  "main": "./dist/index.umd.js",
+  "module": "./dist/index.es.js",
+  "types": "./dist/index.d.ts",
+  "source": "src/index.ts",
+  "dependencies": {
+    "@firecms/core": "^3.0.0-3.0.0-beta.4.pre.1.0",
+    "@firecms/schema_inference": "^3.0.0-3.0.0-beta.4.pre.1.0",
+    "xlsx": "^0.18.5"
+  },
+  "peerDependencies": {
+    "firebase": "^10.7.1",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-router": "^6.22.2",
+    "react-router-dom": "^6.22.2"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.es.js",
+      "require": "./dist/index.umd.js",
+      "types": "./dist/src/index.d.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build && tsc --emitDeclarationOnly -p tsconfig.prod.json",
+    "prepublishOnly": "run-s build",
+    "clean": "rm -rf dist && find ./src -name '*.js' -type f | xargs rm -f",
+    "test": "jest"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app",
+      "react-app/jest"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@testing-library/jest-dom": "^6.4.2",
+    "@types/jest": "^29.5.12",
+    "@types/react": "^18.2.67",
+    "@types/react-dom": "^18.2.22",
+    "@typescript-eslint/eslint-plugin": "^7.3.1",
+    "@typescript-eslint/parser": "^7.3.1",
+    "@vitejs/plugin-react": "^4.2.1",
+    "babel-jest": "^29.7.0",
+    "eslint": "^8.57.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-n": "^16.6.2",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-react": "^7.34.1",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.2",
+    "typescript": "^5.4.2",
+    "vite": "^5.1.6",
+    "vite-plugin-fonts": "^0.7.0"
+  },
+  "jest": {
+    "transform": {
+      "^.+\\.tsx?$": "ts-jest"
+    },
+    "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$",
+    "moduleFileExtensions": [
+      "ts",
+      "tsx",
+      "js",
+      "jsx",
+      "json",
+      "node"
+    ],
+    "moduleNameMapper": {
+      "\\.(css|less)$": "<rootDir>/test/__mocks__/styleMock.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "bfe875c02132b5763ac0c9d8c8a885c791bb5a24"
+}

package/src/components/DataNewPropertiesMapping.tsx

@@ -0,0 +1,128 @@
+import { getPropertyInPath, Property, } from "@firecms/core";
+import {
+    ChevronRightIcon,
+    Select,
+    SelectItem,
+    Table,
+    TableBody,
+    TableCell,
+    TableHeader,
+    TableRow,
+    Typography
+} from "@firecms/ui";
+
+export interface DataPropertyMappingProps {
+    idColumn?: string;
+    headersMapping: Record<string, string | null>;
+    originProperties: Record<string, Property>;
+    destinationProperties: Record<string, Property>;
+    onIdPropertyChanged: (value: string | null) => void;
+    buildPropertyView?: (props: {
+        isIdColumn: boolean,
+        property: Property | null,
+        propertyKey: string | null,
+        importKey: string
+    }) => React.ReactNode;
+}
+
+export function DataNewPropertiesMapping({
+                                             idColumn,
+                                             headersMapping,
+                                             originProperties,
+                                             destinationProperties,
+                                             onIdPropertyChanged,
+                                             buildPropertyView,
+                                         }: DataPropertyMappingProps) {
+
+    return (
+        <>
+
+            <IdSelectField idColumn={idColumn}
+                           headersMapping={headersMapping}
+                           onChange={onIdPropertyChanged}/>
+
+            <Table style={{
+                tableLayout: "fixed"
+            }}>
+                <TableHeader>
+                    <TableCell header={true} style={{ width: "20%" }}>
+                        Column in file
+                    </TableCell>
+                    <TableCell header={true}>
+                    </TableCell>
+                    <TableCell header={true} style={{ width: "75%" }}>
+                        Property
+                    </TableCell>
+                </TableHeader>
+                <TableBody>
+                    {destinationProperties &&
+                        Object.entries(headersMapping)
+                            .map(([importKey, mappedKey]) => {
+                                    const propertyKey = headersMapping[importKey];
+                                    const property = mappedKey ? getPropertyInPath(destinationProperties, mappedKey) as Property : null;
+
+                                    const originProperty = getPropertyInPath(originProperties, importKey) as Property | undefined;
+                                    const originDataType = originProperty ? (originProperty.dataType === "array" && typeof originProperty.of === "object"
+                                            ? `${originProperty.dataType} - ${(originProperty.of as Property).dataType}`
+                                            : originProperty.dataType)
+                                        : undefined;
+                                    return <TableRow key={importKey} style={{ height: "90px" }}>
+                                        <TableCell style={{ width: "20%" }}>
+                                            <Typography variant={"body2"}>{importKey}</Typography>
+                                            {originProperty && <Typography
+                                                variant={"caption"}
+                                                color={"secondary"}
+                                            >{originDataType}</Typography>}
+                                        </TableCell>
+                                        <TableCell>
+                                            <ChevronRightIcon/>
+                                        </TableCell>
+                                        <TableCell className={importKey === idColumn ? "text-center" : undefined}
+                                                   style={{ width: "75%" }}>
+                                            {buildPropertyView?.({
+                                                isIdColumn: importKey === idColumn,
+                                                property,
+                                                propertyKey,
+                                                importKey
+                                            })
+                                            }
+                                        </TableCell>
+                                    </TableRow>;
+                                }
+                            )}
+                </TableBody>
+            </Table>
+        </>
+    );
+}
+
+function IdSelectField({
+                           idColumn,
+                           headersMapping,
+                           onChange
+                       }: {
+    idColumn?: string,
+    headersMapping: Record<string, string | null>;
+    onChange: (value: string | null) => void
+}) {
+    return <div>
+        <Select
+            size={"small"}
+            value={idColumn ?? ""}
+            onChange={(event) => {
+                const value = event.target.value;
+                onChange(value === "none" ? null : value);
+            }}
+            renderValue={(value) => {
+                return <Typography variant={"body2"}>
+                    {value !== "" ? value : "Autogenerate ID"}
+                </Typography>;
+            }}
+            label={"Column that will be used as ID for each document"}>
+            <SelectItem value={"none"}>Autogenerate ID</SelectItem>
+            {Object.entries(headersMapping).map(([key, value]) => {
+                return <SelectItem key={key} value={key}>{key}</SelectItem>;
+            })}
+        </Select>
+    </div>;
+}

package/src/components/ImportFileUpload.tsx

@@ -0,0 +1,34 @@
+import { FileUpload, UploadIcon } from "@firecms/ui";
+import { convertFileToJson } from "../utils/file_to_json";
+import { useSnackbarController } from "@firecms/core";
+
+export function ImportFileUpload({ onDataAdded }: { onDataAdded: (data: object[]) => void }) {
+    const snackbarController = useSnackbarController();
+    return <FileUpload
+        accept={{
+            "text/*": [".csv", ".xls", ".xlsx"],
+            "application/vnd.ms-excel": [".xls", ".xlsx"],
+            "application/msexcel": [".xls", ".xlsx"],
+            "application/vnd.ms-office": [".xls", ".xlsx"],
+            "application/xls": [".xls", ".xlsx"],
+            "application/x-xls": [".xls", ".xlsx"],
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xls", ".xlsx"],
+            "application/json": [".json"],
+        }}
+        preventDropOnDocument={true}
+        size={"small"}
+        maxFiles={1}
+        uploadDescription={<><UploadIcon/>Drag and drop a file here or click to upload</>}
+        onFilesAdded={(files: File[]) => {
+            if (files.length > 0) {
+                convertFileToJson(files[0])
+                    .then((jsonData) => {
+                        onDataAdded(jsonData);
+                    })
+                    .catch((error) => {
+                        console.error("Error parsing file", error);
+                        snackbarController.open({ type: "error", message: error.message });
+                    });
+            }
+        }}/>
+}

package/src/components/ImportNewPropertyFieldPreview.tsx

@@ -0,0 +1,55 @@
+import React from "react";
+import { ErrorBoundary, PropertyConfigBadge, getFieldConfig, Property, useCustomizationController } from "@firecms/core";
+import { EditIcon, IconButton, TextField, } from "@firecms/ui";
+
+export function ImportNewPropertyFieldPreview({
+                                                  propertyKey,
+                                                  property,
+                                                  onEditClick,
+                                                  includeName = true,
+                                                  onPropertyNameChanged,
+                                                  propertyTypeView
+                                              }: {
+    propertyKey: string | null,
+    property: Property | null
+    includeName?: boolean,
+    onEditClick?: () => void,
+    onPropertyNameChanged?: (propertyKey: string, value: string) => void,
+    propertyTypeView?: React.ReactNode
+}) {
+
+    const { propertyConfigs } = useCustomizationController();
+    const widget = property ? getFieldConfig(property, propertyConfigs) : null;
+
+    return <ErrorBoundary>
+        <div
+            className="flex flex-row w-full items-center">
+
+            <div className={"mx-4"}>
+                {propertyTypeView ?? <PropertyConfigBadge propertyConfig={widget ?? undefined}/>}
+            </div>
+
+            <div className="w-full flex flex-col grow">
+
+                <div className={"flex flex-row items-center gap-2"}>
+                    {includeName &&
+                        <TextField
+                            size={"small"}
+                            className={"text-base grow"}
+                            value={property?.name ?? ""}
+                            onChange={(e) => {
+                                if (onPropertyNameChanged && propertyKey)
+                                    onPropertyNameChanged(propertyKey, e.target.value);
+                            }}/>}
+
+                    <IconButton onClick={onEditClick} size={"small"}>
+                        <EditIcon size={"small"}/>
+                    </IconButton>
+                </div>
+
+            </div>
+
+
+        </div>
+    </ErrorBoundary>
+}

package/src/components/ImportSaveInProgress.tsx

@@ -0,0 +1,122 @@
+import { DataSource, Entity, EntityCollection, useDataSource } from "@firecms/core";
+import { Button, CenteredView, CircularProgress, Typography, } from "@firecms/ui";
+import { useEffect, useRef, useState } from "react";
+import { ImportConfig } from "../types";
+
+export function ImportSaveInProgress<C extends EntityCollection>
+({
+     path,
+     importConfig,
+     collection,
+     onImportSuccess
+ }:
+     {
+         path: string,
+         importConfig: ImportConfig,
+         collection: C,
+         onImportSuccess: (collection: C) => void
+     }) {
+
+    const [errorSaving, setErrorSaving] = useState<Error | undefined>(undefined);
+    const dataSource = useDataSource();
+
+    const savingRef = useRef<boolean>(false);
+
+    const [processedEntities, setProcessedEntities] = useState<number>(0);
+
+    function save() {
+
+        if (savingRef.current)
+            return;
+
+        savingRef.current = true;
+
+        saveDataBatch(
+            dataSource,
+            collection,
+            path,
+            importConfig.entities,
+            0,
+            25,
+            setProcessedEntities
+        ).then(() => {
+            onImportSuccess(collection);
+            savingRef.current = false;
+        }).catch((e) => {
+            setErrorSaving(e);
+            savingRef.current = false;
+        });
+    }
+
+    useEffect(() => {
+        save();
+    }, []);
+
+    if (errorSaving) {
+        return (
+            <CenteredView className={"flex flex-col gap-4 items-center"}>
+                <Typography variant={"h6"}>
+                    Error saving data
+                </Typography>
+
+                <Typography variant={"body2"} color={"error"}>
+                    {errorSaving.message}
+                </Typography>
+                <Button
+                    onClick={save}
+                    variant={"outlined"}>
+                    Retry
+                </Button>
+            </CenteredView>
+        );
+    }
+
+    return (
+        <CenteredView className={"flex flex-col gap-4 items-center"}>
+            <CircularProgress/>
+
+            <Typography variant={"h6"}>
+                Saving data
+            </Typography>
+
+            <Typography variant={"body2"}>
+                {processedEntities}/{importConfig.entities.length} entities saved
+            </Typography>
+
+            <Typography variant={"caption"}>
+                Do not close this tab or the import will be interrupted.
+            </Typography>
+
+        </CenteredView>
+    );
+
+}
+
+function saveDataBatch(dataSource: DataSource,
+                       collection: EntityCollection,
+                       path: string,
+                       data: Partial<Entity<any>>[],
+                       offset = 0,
+                       batchSize = 25,
+                       onProgressUpdate: (progress: number) => void): Promise<void> {
+
+    console.debug("Saving imported data", offset, batchSize);
+
+    const batch = data.slice(offset, offset + batchSize);
+    return Promise.all(batch.map(d =>
+        dataSource.saveEntity({
+            path,
+            values: d.values,
+            entityId: d.id,
+            collection,
+            status: "new"
+        })))
+        .then(() => {
+            if (offset + batchSize < data.length) {
+                onProgressUpdate(offset + batchSize);
+                return saveDataBatch(dataSource, collection, path, data, offset + batchSize, batchSize, onProgressUpdate);
+            }
+            onProgressUpdate(data.length);
+            return Promise.resolve();
+        });
+}

package/src/components/index.ts

@@ -0,0 +1,4 @@
+export * from "./DataNewPropertiesMapping";
+export * from "./ImportFileUpload";
+export * from "./ImportNewPropertyFieldPreview";
+export * from "./ImportSaveInProgress";