@vertesia/ui 0.60.0 → 0.61.0

package/src/features/store/objects/upload/index.ts

@@ -0,0 +1,3 @@
+export * from "./DocumentUploadModal";
+export * from "./useSmartFileUploadProcessing";
+export * from "./useUploadHandler";

package/src/features/store/objects/upload/useSmartFileUploadProcessing.ts

@@ -0,0 +1,250 @@
+import { ContentObjectItem } from "@vertesia/common";
+import { useToast } from "@vertesia/ui/core";
+import { useUserSession } from "@vertesia/ui/session";
+import { Md5 } from "ts-md5";
+
+/**
+ * Types of actions that can be taken with a file
+ */
+export enum FileUploadAction {
+    CREATE = "create", // New document, not in system
+    SKIP = "skip", // Document exists, identical, no action needed
+    UPDATE = "update", // Document exists but needs updating
+}
+
+/**
+ * Interface for file metadata with hash information
+ */
+export interface FileWithMetadata {
+    file: File;
+    hash?: string;
+    name: string;
+    size: number;
+    location?: string;
+    // Optional existing object ID if found
+    existingId?: string;
+    // Action to take with this file
+    action?: FileUploadAction;
+    // Any additional metadata needed
+    metadata?: Record<string, any>;
+}
+
+/**
+ * Calculates the MD5 hash of a file
+ * @param file The file to hash
+ * @returns Promise with the MD5 hash of the file
+ */
+async function calculateFileHash(file: File): Promise<string | undefined> {
+    try {
+        const arrayBuffer = await file.arrayBuffer();
+        const md5 = new Md5();
+        md5.appendByteArray(new Uint8Array(arrayBuffer));
+        const hash = md5.end();
+        return hash?.toString();
+    } catch (error) {
+        console.error("Error calculating file hash:", error);
+        return undefined;
+    }
+}
+
+/**
+ * Prepares files by calculating their hashes and metadata
+ * @param files Array of files to process
+ * @param selectedFolder Optional folder location for the files
+ * @returns Promise with array of files with their metadata
+ */
+async function prepareFilesWithMetadata(files: File[], selectedFolder?: string | null): Promise<FileWithMetadata[]> {
+    const filesWithMetadata: FileWithMetadata[] = [];
+
+    // Process files in batches to avoid overwhelming the browser
+    const batchSize = 5;
+    for (let i = 0; i < files.length; i += batchSize) {
+        const batch = files.slice(i, i + batchSize);
+
+        // Process this batch in parallel
+        const metadataResults = await Promise.all(
+            batch.map(async (file) => {
+                // Determine the location for this file
+                let location = selectedFolder || undefined;
+
+                // If file has relative path, use it to determine location
+                const hasRelativePath = !!(file as any).webkitRelativePath;
+                if (hasRelativePath) {
+                    const relativePath = (file as any).webkitRelativePath;
+                    const pathParts = relativePath.split("/");
+
+                    if (pathParts.length > 1) {
+                        // Extract folder path (everything except the filename)
+                        const folderPath = pathParts.slice(0, -1).join("/");
+
+                        // Combine with selected folder if any
+                        location = selectedFolder ? `${selectedFolder}/${folderPath}` : folderPath;
+                    }
+                }
+
+                // Calculate hash for content comparison
+                const hash = await calculateFileHash(file);
+
+                return {
+                    file,
+                    hash,
+                    name: file.name,
+                    size: file.size,
+                    location,
+                };
+            }),
+        );
+
+        filesWithMetadata.push(...metadataResults);
+    }
+
+    return filesWithMetadata;
+}
+
+/**
+ * Hook to check if documents need to be created, updated, or skipped
+ * @param collectionId Optional collection ID to check within
+ * @returns Object with functions to process documents
+ */
+export function useSmartFileUploadProcessing() {
+    const { client } = useUserSession();
+    const toast = useToast();
+
+    /**
+     * Check files to determine if they need to be created, updated, or skipped
+     * @param files Array of files to check
+     * @param selectedFolder Optional folder location
+     * @param collectionId limit the check to a collection
+     * @returns Promise with information about actions to take for each file
+     */
+    const prepareFilesForUpload = async (
+        files: File[],
+        selectedFolder?: string | null,
+        limitToCollectionId?: string,
+    ): Promise<FileWithMetadata[]> => {
+        try {
+            // First, prepare all the files with their metadata
+            console.log(`Preparing metadata for ${files.length} files...`);
+            const filesWithMetadata = await prepareFilesWithMetadata(files, selectedFolder);
+
+            const identifyExistingHash = async () => {
+                const query = {
+                    "content.etag": { $in: filesWithMetadata.map((file) => file.hash) },
+                };
+
+                let res: ContentObjectItem[];
+
+                if (limitToCollectionId) {
+                    res = await client.store.collections.searchMembers(limitToCollectionId, {
+                        query: {
+                            match: query,
+                        },
+                        select: undefined,
+                    });
+                } else {
+                    res = await client.store.objects.search({
+                        query: { match: query },
+                        select: undefined,
+                    });
+                }
+
+                for (const doc of res) {
+                    const file = filesWithMetadata.find((f) => f.hash === doc.content.etag);
+                    if (file) {
+                        file.existingId = doc.id;
+                        file.action = FileUploadAction.SKIP;
+                    }
+                }
+            };
+
+            /**
+             * Find what file are present based on location and name
+             */
+            const identifyExistingIds = async () => {
+                const unskippedFiles = filesWithMetadata.filter((file) => file.action !== FileUploadAction.SKIP);
+                const allLocations = unskippedFiles.map((file) => file.location);
+                const uniqueLocations = Array.from(new Set(allLocations));
+
+                const queries = [];
+                for (const location of uniqueLocations) {
+                    const namesInLocation = unskippedFiles
+                        .filter((file) => file.location === location)
+                        .map((file) => file.name);
+                    const query = {
+                        location: location ?? "",
+                        "content.name": { $in: namesInLocation },
+                    };
+                    if (limitToCollectionId) {
+                        const res = client.store.collections.searchMembers(limitToCollectionId, {
+                            query: {
+                                match: query,
+                            },
+                            select: undefined,
+                        });
+                        queries.push(res);
+                    } else {
+                        const res = client.store.objects.find({
+                            query: {
+                                match: query,
+                            },
+                            select: undefined,
+                        });
+                        queries.push(res);
+                    }
+                }
+
+                const results = (await Promise.all(queries)).flat();
+                console.log(`Found ${results.length} document to update`, results);
+
+                //update fileWithMetadata
+                for (const doc of results) {
+                    const file = filesWithMetadata.find(
+                        //name must be the same, and location must the same if present and if not, must be empty string
+                        (f) =>
+                            f.name === doc.content.name &&
+                            (f.location ? f.location === doc.location : doc.location === ""),
+                    );
+                    if (file) {
+                        file.existingId = doc.id;
+                        file.action = FileUploadAction.UPDATE;
+                    }
+                }
+                console.log(
+                    `Reconciled ${filesWithMetadata.filter((f) => f.action === FileUploadAction.UPDATE).length}`,
+                );
+            };
+
+            await identifyExistingHash();
+            await identifyExistingIds();
+
+            //set create flag on remaining files
+            filesWithMetadata.forEach((f) => {
+                if (!f.action) {
+                    f.action = FileUploadAction.CREATE;
+                }
+            });
+
+            // Log the results
+            console.log("Document processing check results:", {
+                totalFiles: files.length,
+                toCreate: filesWithMetadata.filter((f) => f.action === FileUploadAction.CREATE).length,
+                toUpdate: filesWithMetadata.filter((f) => f.action === FileUploadAction.UPDATE).length,
+                toSkip: filesWithMetadata.filter((f) => f.action === FileUploadAction.SKIP).length,
+            });
+
+            return filesWithMetadata;
+        } catch (error: any) {
+            toast({
+                title: "Error in file upload processing check",
+                status: "error",
+                description: error.message,
+            });
+            console.log("Error in file upload processing check", error);
+            throw new Error("Error in file upload processing check: " + error.message);
+        }
+    };
+
+    return {
+        checkDocumentProcessing: prepareFilesForUpload,
+    };
+}

package/src/features/store/objects/upload/useUploadHandler.ts

@@ -0,0 +1,325 @@
+import { useToast } from "@vertesia/ui/core";
+import { useUserSession } from "@vertesia/ui/session";
+import { useDocumentSearch } from "../search/DocumentSearchContext";
+import { FileUploadAction, useSmartFileUploadProcessing } from "./useSmartFileUploadProcessing";
+
+
+/**
+ * Hook configuration for useUploadHandler
+ */
+export interface UploadHandlerOptions {
+    /**
+     * Callback to be called after upload is done with the created object IDs
+     */
+    onUploadDone: (objectIds: string[]) => Promise<void>;
+}
+
+/**
+ * Result of the upload process
+ */
+export interface DocumentUploadResult {
+    /** Whether the upload was successful overall */
+    success: boolean;
+    /** IDs of all processed objects (both newly uploaded and existing) */
+    objectIds: string[];
+    /** Newly uploaded files information */
+    uploadedFiles: UploadedFileInfo[];
+    /** Files that were skipped (already existed) */
+    skippedFiles: UploadedFileInfo[];
+    /** Files that failed to upload */
+    failedFiles: UploadedFileInfo[];
+}
+
+/**
+ * Information about a successfully uploaded file
+ */
+interface UploadedFileInfo {
+    /** The ID of the uploaded object */
+    id?: string;
+    /** The name of the file */
+    name: string;
+    /** The name of the file */
+    location?: string;
+    /** The status of the upload */
+    status: "created" | "updated" | "skipped" | "failed";
+    /** The type of the object */
+    type: string | null;
+    /** Has there been any Error */
+    error?: string;
+}
+
+/**
+ * Hook for handling file uploads to the store
+ *
+ * @param options Configuration options for the upload handler
+ * @returns Upload handler function that takes files and type, and returns upload results
+ */
+export function useDocumentUploadHandler(options: UploadHandlerOptions | ((objectIds: string[]) => Promise<void>)) {
+    // Handle both object and legacy function format for backward compatibility
+    const onUploadDone = typeof options === "function" ? options : options.onUploadDone;
+    const { client, project: projectRef, store } = useUserSession();
+    const search = useDocumentSearch();
+    const toast = useToast();
+    const { checkDocumentProcessing } = useSmartFileUploadProcessing();
+
+    return async (files: File[], type: string | null, collectionId?: string): Promise<DocumentUploadResult> => {
+        // Initialize result object
+        const result: DocumentUploadResult = {
+            success: false,
+            objectIds: [],
+            uploadedFiles: [],
+            skippedFiles: [],
+            failedFiles: [],
+        };
+
+        if (!projectRef) {
+            toast({
+                status: "error",
+                title: "No project selected",
+                duration: 3000,
+            });
+            return result;
+        }
+
+        if (!files || files.length === 0) {
+            toast({
+                status: "warning",
+                title: "No files selected",
+                duration: 3000,
+            });
+            return result;
+        }
+
+        const filesToUpload = Array.from(files).filter((file) => file.size > 0);
+
+        if (filesToUpload.length === 0) {
+            toast({
+                status: "error",
+                title: "No valid files selected",
+                description: "Please select files with valid content.",
+                duration: 5000,
+            });
+            return result;
+        }
+
+        const skippedFilesInfo: UploadedFileInfo[] = [];
+        const uploadedFilesInfo: UploadedFileInfo[] = [];
+        const failedFilesInfo: UploadedFileInfo[] = [];
+
+        console.log(`Processing ${filesToUpload.length} files with smart file processing...`);
+
+        // Determine if what need to be updated
+        const filesWithMetadata = await checkDocumentProcessing(filesToUpload, null, collectionId);
+
+        // Create statistics for user feedback
+        const toCreate = filesWithMetadata.filter((f) => f.action === FileUploadAction.CREATE).length;
+        const toUpdate = filesWithMetadata.filter((f) => f.action === FileUploadAction.UPDATE).length;
+        const toSkip = filesWithMetadata.filter((f) => f.action === FileUploadAction.SKIP).length;
+
+        // Show user feedback about processed files
+        toast({
+            title: "Files analyzed",
+            description: `${filesToUpload.length} file(s): ${toCreate} new, ${toUpdate} to update, ${toSkip} to skip`,
+            status: "info",
+            duration: 4000,
+        });
+
+        // Process SKIP files - add them to skipped list and result IDs
+        for (const fileInfo of filesWithMetadata.filter((f) => f.action === FileUploadAction.SKIP && f.existingId)) {
+            skippedFilesInfo.push({
+                id: fileInfo.existingId,
+                name: fileInfo.name,
+                type,
+                status: "skipped",
+                location: fileInfo.location,
+            });
+        }
+
+        // Collect files for processing (both CREATE and UPDATE)
+        const filesToProcess = filesWithMetadata.filter(
+            (f) => (f.action === FileUploadAction.CREATE || f.action === FileUploadAction.UPDATE) && f.file,
+        );
+
+        if (filesToProcess.length > 0) {
+            console.log(
+                `Processing ${filesToProcess.length} files (${filesToProcess.filter((f) => f.action === FileUploadAction.CREATE).length
+                } create, ${filesToProcess.filter((f) => f.action === FileUploadAction.UPDATE).length} update)...`,
+            );
+
+            // Create batches of 50 files
+            const batchSize = 50;
+            const processBatches = [];
+
+            for (let i = 0; i < filesToProcess.length; i += batchSize) {
+                processBatches.push(filesToProcess.slice(i, i + batchSize));
+            }
+
+            // Process each batch
+            for (let batchIndex = 0; batchIndex < processBatches.length; batchIndex++) {
+                const batch = processBatches[batchIndex];
+                const processedCount = batchIndex * batchSize;
+
+                console.log(`Processing batch ${batchIndex + 1}/${processBatches.length} (${batch.length} files)`);
+
+                // Show progress for larger file sets
+                if (filesToProcess.length > batchSize) {
+                    toast({
+                        title: "Processing files",
+                        description: `Processed ${processedCount}/${filesToProcess.length} files...`,
+                        status: "info",
+                        duration: 2000,
+                    });
+                }
+
+                // Process this batch in parallel
+                const processResults = await Promise.all(
+                    batch.map(async (fileInfo) => {
+                        try {
+                            if (fileInfo.action === FileUploadAction.UPDATE && fileInfo.existingId) {
+                                // UPDATE - Update the existing object
+                                const updateResult = await client.store.objects.update(
+                                    fileInfo.existingId,
+                                    {
+                                        type: type || undefined,
+                                        content: fileInfo.file,
+                                        location: fileInfo.location,
+                                        name: fileInfo.name,
+                                    },
+                                    {
+                                        createRevision: true,
+                                        revisionLabel: "upload on " + new Date().toISOString(),
+                                    },
+                                );
+
+                                // Use a separate data structure for updated files
+                                uploadedFilesInfo.push({
+                                    id: fileInfo.existingId,
+                                    name: fileInfo.name,
+                                    type,
+                                    status: "updated",
+                                    location: fileInfo.location,
+                                });
+
+                                // Add to result IDs
+                                result.objectIds.push(fileInfo.existingId);
+
+                                return {
+                                    success: true,
+                                    id: updateResult.id,
+                                    name: fileInfo.name,
+                                    action: "update",
+                                };
+                            } else {
+                                // CREATE - Create a new object
+                                // If collectionId is provided, pass it as an option
+                                const options = collectionId ? { collection_id: collectionId } : undefined;
+                                const createResult = await store.objects.create(
+                                    {
+                                        type: type || undefined,
+                                        content: fileInfo.file,
+                                        location:
+                                            fileInfo.location || fileInfo.file.webkitRelativePath || fileInfo.name,
+                                    },
+                                    options,
+                                );
+
+                                // Track successful upload
+                                uploadedFilesInfo.push({
+                                    id: createResult.id,
+                                    name: fileInfo.name,
+                                    type,
+                                    status: "created",
+                                    location: fileInfo.location,
+                                });
+                            }
+                        } catch (error: any) {
+                            console.error(`Failed to process file ${fileInfo.name}:`, error);
+
+                            // Track failed upload
+                            failedFilesInfo.push({
+                                name: fileInfo.name,
+                                error: error.message || "Unknown error",
+                                status: "failed",
+                                location: fileInfo.location,
+                                type,
+                            });
+
+                            toast({
+                                status: "error",
+                                title: `Processing failed for ${fileInfo.name}`,
+                                description: error.message,
+                                duration: 4000,
+                            });
+                        }
+                    }),
+                );
+
+                // Log batch results
+                const creates = uploadedFilesInfo.filter((f) => f.status === "created");
+                const updates = uploadedFilesInfo.filter((f) => f.status === "updated");
+                const failures = uploadedFilesInfo.filter((f) => f.status === "failed");
+
+                console.log(`Batch ${batchIndex + 1} results:`, {
+                    creates,
+                    updates,
+                    failures,
+                    total: processResults.length,
+                });
+            }
+        }
+
+        const uploadedCount = uploadedFilesInfo.filter((f) => f.status === "created").length;
+        const updatedCount = uploadedFilesInfo.filter((f) => f.status === "updated").length;
+        const failedCount = uploadedFilesInfo.filter((f) => f.status === "failed").length;
+        const skippedCount = uploadedFilesInfo.filter((f) => f.status === "skipped").length;
+
+        // Call the original callback with all object IDs and additional metadata
+        if (onUploadDone) {
+            // Log the actual upload count for debugging
+            console.log("Upload complete:", {
+                totalObjectIds: result.objectIds.length,
+                uploadedFiles: uploadedCount,
+                updatedFiles: updatedCount,
+                skippedFiles: skippedCount,
+                failedFiles: failedCount,
+            });
+
+            // Call the callback with all object IDs
+            await onUploadDone(result.objectIds);
+        }
+
+        // Create a success message that includes information about all file operations
+        let statusMessage = "";
+
+        if (uploadedCount > 0) {
+            statusMessage += `${uploadedCount} file${uploadedCount !== 1 ? "s" : ""} uploaded`;
+        }
+
+        if (updatedCount > 0) {
+            statusMessage += statusMessage ? ", " : "";
+            statusMessage += `${updatedCount} file${updatedCount !== 1 ? "s" : ""} updated`;
+        }
+
+        if (skippedCount > 0) {
+            statusMessage += statusMessage ? ", " : "";
+            statusMessage += `${skippedCount} file${skippedCount !== 1 ? "s" : ""} skipped (already existed)`;
+        }
+
+        if (failedCount > 0) {
+            statusMessage += statusMessage ? ", " : "";
+            statusMessage += `${failedCount} file${failedCount !== 1 ? "s" : ""} failed`;
+        }
+
+        if (statusMessage) {
+            toast({
+                status: failedCount > 0 ? "warning" : "success",
+                title: statusMessage,
+                duration: 4000,
+            });
+        }
+
+        // Refresh search results
+        search.search();
+        return result;
+    };
+}