@librechat/data-schemas 0.0.33 → 0.0.35

package/dist/index.cjs

@@ -2246,9 +2246,10 @@ const rolePermissionsSchema = new mongoose.Schema({
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
     },
     [librechatDataProvider.PermissionTypes.PROMPTS]: {
-        [librechatDataProvider.Permissions.SHARED_GLOBAL]: { type: Boolean },
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
         [librechatDataProvider.Permissions.CREATE]: { type: Boolean },
+        [librechatDataProvider.Permissions.SHARE]: { type: Boolean },
+        [librechatDataProvider.Permissions.SHARE_PUBLIC]: { type: Boolean },
     },
     [librechatDataProvider.PermissionTypes.MEMORIES]: {
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
@@ -2258,9 +2259,10 @@ const rolePermissionsSchema = new mongoose.Schema({
         [librechatDataProvider.Permissions.OPT_OUT]: { type: Boolean },
     },
     [librechatDataProvider.PermissionTypes.AGENTS]: {
-        [librechatDataProvider.Permissions.SHARED_GLOBAL]: { type: Boolean },
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
         [librechatDataProvider.Permissions.CREATE]: { type: Boolean },
+        [librechatDataProvider.Permissions.SHARE]: { type: Boolean },
+        [librechatDataProvider.Permissions.SHARE_PUBLIC]: { type: Boolean },
     },
     [librechatDataProvider.PermissionTypes.MULTI_CONVO]: {
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
@@ -2292,6 +2294,7 @@ const rolePermissionsSchema = new mongoose.Schema({
         [librechatDataProvider.Permissions.USE]: { type: Boolean },
         [librechatDataProvider.Permissions.CREATE]: { type: Boolean },
         [librechatDataProvider.Permissions.SHARE]: { type: Boolean },
+        [librechatDataProvider.Permissions.SHARE_PUBLIC]: { type: Boolean },
     },
 }, { _id: false });
 const roleSchema = new mongoose.Schema({
@@ -2857,8 +2860,8 @@ const createMeiliMongooseModel = ({ index, attributesToIndex, syncOptions, }) =>
          * Get the current sync progress
          */
         static async getSyncProgress() {
-            const totalDocuments = await this.countDocuments();
-            const indexedDocuments = await this.countDocuments({ _meiliIndex: true });
+            const totalDocuments = await this.countDocuments({ expiredAt: null });
+            const indexedDocuments = await this.countDocuments({ expiredAt: null, _meiliIndex: true });
             return {
                 totalProcessed: indexedDocuments,
                 totalDocuments,
@@ -2866,93 +2869,84 @@ const createMeiliMongooseModel = ({ index, attributesToIndex, syncOptions, }) =>
             };
         }
         /**
-         * Synchronizes the data between the MongoDB collection and the MeiliSearch index.
-         * Now uses streaming and batching to reduce memory usage.
-         */
-        static async syncWithMeili(options) {
+         * Synchronizes data between the MongoDB collection and the MeiliSearch index by
+         * incrementally indexing only documents where `expiredAt` is `null` and `_meiliIndex` is `false`
+         * (i.e., non-expired documents that have not yet been indexed).
+         * */
+        static async syncWithMeili() {
+            const startTime = Date.now();
+            const { batchSize, delayMs } = syncConfig;
+            const collectionName = primaryKey === 'messageId' ? 'messages' : 'conversations';
+            logger.info(`[syncWithMeili] Starting sync for ${collectionName} with batch size ${batchSize}`);
+            // Get approximate total count for raw estimation, the sync should not overcome this number
+            const approxTotalCount = await this.estimatedDocumentCount();
+            logger.info(`[syncWithMeili] Approximate total number of all ${collectionName}: ${approxTotalCount}`);
             try {
-                const startTime = Date.now();
-                const { batchSize, delayMs } = syncConfig;
-                logger.info(`[syncWithMeili] Starting sync for ${primaryKey === 'messageId' ? 'messages' : 'conversations'} with batch size ${batchSize}`);
-                // Build query with resume capability
-                // Do not sync TTL documents
-                const query = { expiredAt: null };
-                if (options === null || options === void 0 ? void 0 : options.resumeFromId) {
-                    query._id = { $gt: options.resumeFromId };
-                }
-                // Get total count for progress tracking
-                const totalCount = await this.countDocuments(query);
-                let processedCount = 0;
                 // First, handle documents that need to be removed from Meili
+                logger.info(`[syncWithMeili] Starting cleanup of Meili index ${index.uid} before sync`);
                 await this.cleanupMeiliIndex(index, primaryKey, batchSize, delayMs);
-                // Process MongoDB documents in batches using cursor
-                const cursor = this.find(query)
-                    .select(attributesToIndex.join(' ') + ' _meiliIndex')
-                    .sort({ _id: 1 })
-                    .batchSize(batchSize)
-                    .cursor();
-                const format = (doc) => _.omitBy(_.pick(doc, attributesToIndex), (v, k) => k.startsWith('$'));
-                let documentBatch = [];
-                let updateOps = [];
-                // Process documents in streaming fashion
-                for await (const doc of cursor) {
-                    const typedDoc = doc.toObject();
-                    const formatted = format(typedDoc);
-                    // Check if document needs indexing
-                    if (!typedDoc._meiliIndex) {
-                        documentBatch.push(formatted);
-                        updateOps.push({
-                            updateOne: {
-                                filter: { _id: typedDoc._id },
-                                update: { $set: { _meiliIndex: true } },
-                            },
-                        });
+                logger.info(`[syncWithMeili] Completed cleanup of Meili index: ${index.uid}`);
+            }
+            catch (error) {
+                logger.error('[syncWithMeili] Error during cleanup Meili before sync:', error);
+                throw error;
+            }
+            let processedCount = 0;
+            let hasMore = true;
+            while (hasMore) {
+                const query = {
+                    expiredAt: null,
+                    _meiliIndex: false,
+                };
+                try {
+                    const documents = await this.find(query)
+                        .select(attributesToIndex.join(' ') + ' _meiliIndex')
+                        .limit(batchSize)
+                        .lean();
+                    // Check if there are more documents to process
+                    if (documents.length === 0) {
+                        logger.info('[syncWithMeili] No more documents to process');
+                        break;
                     }
-                    processedCount++;
-                    // Process batch when it reaches the configured size
-                    if (documentBatch.length >= batchSize) {
-                        await this.processSyncBatch(index, documentBatch, updateOps);
-                        documentBatch = [];
-                        updateOps = [];
-                        // Log progress
-                        const progress = Math.round((processedCount / totalCount) * 100);
-                        logger.info(`[syncWithMeili] Progress: ${progress}% (${processedCount}/${totalCount})`);
-                        // Add delay to prevent overwhelming resources
-                        if (delayMs > 0) {
-                            await new Promise((resolve) => setTimeout(resolve, delayMs));
-                        }
+                    // Process the batch
+                    await this.processSyncBatch(index, documents);
+                    processedCount += documents.length;
+                    logger.info(`[syncWithMeili] Processed: ${processedCount}`);
+                    if (documents.length < batchSize) {
+                        hasMore = false;
+                    }
+                    // Add delay to prevent overwhelming resources
+                    if (hasMore && delayMs > 0) {
+                        await new Promise((resolve) => setTimeout(resolve, delayMs));
                     }
                 }
-                // Process remaining documents
-                if (documentBatch.length > 0) {
-                    await this.processSyncBatch(index, documentBatch, updateOps);
+                catch (error) {
+                    logger.error('[syncWithMeili] Error processing documents batch:', error);
+                    throw error;
                 }
-                const duration = Date.now() - startTime;
-                logger.info(`[syncWithMeili] Completed sync for ${primaryKey === 'messageId' ? 'messages' : 'conversations'} in ${duration}ms`);
-            }
-            catch (error) {
-                logger.error('[syncWithMeili] Error during sync:', error);
-                throw error;
             }
+            const duration = Date.now() - startTime;
+            logger.info(`[syncWithMeili] Completed sync for ${collectionName}. Processed ${processedCount} documents in ${duration}ms`);
         }
         /**
          * Process a batch of documents for syncing
          */
-        static async processSyncBatch(index, documents, updateOps) {
+        static async processSyncBatch(index, documents) {
             if (documents.length === 0) {
                 return;
             }
+            // Format documents for MeiliSearch
+            const formattedDocs = documents.map((doc) => _.omitBy(_.pick(doc, attributesToIndex), (_v, k) => k.startsWith('$')));
             try {
                 // Add documents to MeiliSearch
-                await index.addDocuments(documents);
+                await index.addDocumentsInBatches(formattedDocs);
                 // Update MongoDB to mark documents as indexed
-                if (updateOps.length > 0) {
-                    await this.collection.bulkWrite(updateOps);
-                }
+                const docsIds = documents.map((doc) => doc._id);
+                await this.updateMany({ _id: { $in: docsIds } }, { $set: { _meiliIndex: true } });
             }
             catch (error) {
                 logger.error('[processSyncBatch] Error processing batch:', error);
-                // Don't throw - allow sync to continue with other documents
+                throw error;
             }
         }
         /**
@@ -2977,10 +2971,14 @@ const createMeiliMongooseModel = ({ index, attributesToIndex, syncOptions, }) =>
                     // Delete documents that don't exist in MongoDB
                     const toDelete = meiliIds.filter((id) => !existingIds.has(id));
                     if (toDelete.length > 0) {
-                        await Promise.all(toDelete.map((id) => index.deleteDocument(id)));
+                        await index.deleteDocuments(toDelete.map(String));
                         logger.debug(`[cleanupMeiliIndex] Deleted ${toDelete.length} orphaned documents`);
                     }
-                    offset += batchSize;
+                    // if fetch documents request returns less documents than limit, all documents are processed
+                    if (batch.results.length < batchSize) {
+                        break;
+                    }
+                    offset += batchSize - toDelete.length;
                     // Add delay between batches
                     if (delayMs > 0) {
                         await new Promise((resolve) => setTimeout(resolve, delayMs));
@@ -5078,10 +5076,10 @@ function createMCPServerMethods(mongoose) {
     }
     /**
      * Find an MCP server by serverName
-     * @param serverName - The MCP server ID
+     * @param serverName - The unique server name identifier
      * @returns The MCP server document or null
      */
-    async function findMCPServerById(serverName) {
+    async function findMCPServerByServerName(serverName) {
         const MCPServer = mongoose.models.MCPServer;
         return await MCPServer.findOne({ serverName }).lean();
     }
@@ -5210,7 +5208,7 @@ function createMCPServerMethods(mongoose) {
     }
     return {
         createMCPServer,
-        findMCPServerById,
+        findMCPServerByServerName,
         findMCPServerByObjectId,
         findMCPServersByAuthor,
         getListMCPServersByIds,