libre-webui 0.25.0 → 0.26.0

package/backend/dist/services/documentService.js

@@ -18,6 +18,10 @@ import crypto from 'node:crypto';
 import { Readable } from 'node:stream';
 import { v4 as uuidv4 } from 'uuid';
 import storageService from '../storage.js';
+import { getPersistence } from '../persistence/index.js';
+import { encryptionService } from './encryptionService.js';
+import { listGrantsForActor } from './resourceGrantService.js';
+import { grantedResourceIdsFor, sharedMetaFor, } from './sharedResourceAccess.js';
 import { configurePlatformStorageRuntime, getPlatformStorageRuntime, } from '../platform/storage/index.js';
 import { MAX_VECTOR_RECORDS_PER_UPSERT, MAX_VECTOR_RESOURCE_INDEX_ENTRIES, } from '../platform/storage/vectorStore.js';
 import { aggregateDocumentIndexRevision, createDocumentIndexMetadata, DOCUMENT_CHUNKER_VERSION, DOCUMENT_INDEX_METADATA_KEY, DOCUMENT_VECTOR_NAMESPACE, DOCUMENT_VECTOR_VERSION, documentChunkSourceRevision as sourceRevision, documentEmbeddingDimensions as embeddingDimensions, documentIndexMatchesSpec, embeddedDocumentChunks as embeddedChunks, readDocumentIndexMetadata as readDocumentIndexMetadataValue, } from '../platform/storage/documentVectorIndex.js';
@@ -29,6 +33,8 @@ import { transactionalResourceDeletionEnqueuer } from '../platform/jobs/resource
 import preferencesService from './preferencesService.js';
 import { createLogger } from '../utils/logger.js';
 import { throwIfChatGenerationCancelled } from '../utils/chatCancellation.js';
+import { reciprocalRankFusion, scoreCandidatesBm25, } from '../utils/hybridRetrieval.js';
+import { extractDocumentContentByType, readDocumentSegments, resolveDocumentFileType, resolveSegmentLabel, } from '../utils/documentExtraction.js';
 const logger = createLogger('documents');
 const DOCUMENT_BLOB_PURPOSE = 'document.source';
 const DOCUMENT_POST_VECTOR_FAULT_MARKER = 'LIBRE_DOCUMENT_POST_VECTOR_KILL';
@@ -248,31 +254,29 @@ const getPdfjsLib = async () => {
     }
     return pdfjsLib;
 };
-const extractDocumentContent = async (fileBuffer, mimeType, signal) => {
+/**
+ * Segment maps above this length would bloat the encrypted metadata column;
+ * a source under the extraction size cap never legitimately reaches it.
+ */
+const MAX_DOCUMENT_SEGMENTS = 5000;
+const extractDocumentContent = async (fileName, fileBuffer, mimeType, signal) => {
     if (signal?.aborted)
         throw signal.reason;
-    if (mimeType === 'text/plain') {
-        return { content: fileBuffer.toString('utf8'), fileType: 'txt' };
-    }
-    if (mimeType !== 'application/pdf') {
+    const fileType = resolveDocumentFileType(fileName, mimeType);
+    if (!fileType) {
         throw new Error(`Unsupported file type: ${mimeType}`);
     }
-    const pdfLib = await getPdfjsLib();
-    const pdfDocument = await pdfLib.getDocument({
-        data: new Uint8Array(fileBuffer),
-    }).promise;
-    let textContent = '';
-    for (let pageNum = 1; pageNum <= pdfDocument.numPages; pageNum += 1) {
-        if (signal?.aborted)
-            throw signal.reason;
-        const page = await pdfDocument.getPage(pageNum);
-        const pageText = await page.getTextContent();
-        textContent += `${pageText.items
-            .map(item => ('str' in item ? item.str : ''))
-            .join(' ')}\n\n`;
-    }
-    return { content: textContent.trim(), fileType: 'pdf' };
+    const extracted = await extractDocumentContentByType(fileBuffer, fileType, {
+        signal,
+        ...(fileType === 'pdf' ? { pdfLib: await getPdfjsLib() } : {}),
+    });
+    return {
+        content: extracted.content,
+        fileType,
+        segments: extracted.segments.slice(0, MAX_DOCUMENT_SEGMENTS),
+    };
 };
+const documentContentSha256 = (fileBuffer) => crypto.createHash('sha256').update(fileBuffer).digest('hex');
 export class DocumentService {
     async captureEmbeddingExecutionSpec(userId) {
         const settings = (await preferencesService.getPreferences(userId))
@@ -333,23 +337,36 @@ export class DocumentService {
      * if the SQL transaction cannot commit.
      */
     async queueDocumentProcessing(fileName, fileBuffer, mimeType, userId, sessionId, signal) {
-        if (mimeType !== 'text/plain' && mimeType !== 'application/pdf') {
+        const resolvedType = resolveDocumentFileType(fileName, mimeType);
+        if (!resolvedType) {
             throw new Error(`Unsupported file type: ${mimeType}`);
         }
         if (signal?.aborted)
             throw signal.reason;
+        const contentSha256 = documentContentSha256(fileBuffer);
+        const duplicate = await this.findDuplicateDocument(userId, sessionId, contentSha256);
+        if (duplicate) {
+            const existingJob = await getDurableJobRuntime().service.getByIdempotency(userId, DOCUMENT_INGEST_IDEMPOTENCY_SCOPE, duplicate.id);
+            if (existingJob) {
+                return {
+                    document: duplicate,
+                    jobId: existingJob.id,
+                    deduplicated: true,
+                };
+            }
+        }
         const documentId = uuidv4();
         const now = Date.now();
         const document = {
             id: documentId,
             filename: fileName,
             content: '',
-            fileType: mimeType === 'application/pdf' ? 'pdf' : 'txt',
+            fileType: resolvedType,
             size: fileBuffer.length,
             ...(sessionId ? { sessionId } : {}),
             uploadedAt: now,
             createdAt: now,
-            metadata: { processingStatus: 'queued' },
+            metadata: { processingStatus: 'queued', contentSha256 },
         };
         const platform = getPlatformStorageRuntime();
         const sourceBlob = await platform.blobStore.put({
@@ -417,7 +434,7 @@ export class DocumentService {
     async processDocument(fileName, fileBuffer, mimeType, userId, sessionId) {
         const documentId = uuidv4();
         try {
-            const { content, fileType } = await extractDocumentContent(fileBuffer, mimeType);
+            const { content, fileType, segments } = await extractDocumentContent(fileName, fileBuffer, mimeType);
             let document = {
                 id: documentId,
                 filename: fileName,
@@ -426,6 +443,10 @@ export class DocumentService {
                 size: fileBuffer.length,
                 sessionId,
                 uploadedAt: Date.now(),
+                metadata: {
+                    contentSha256: documentContentSha256(fileBuffer),
+                    ...(segments.length > 0 ? { segments } : {}),
+                },
             };
             const spec = await this.captureEmbeddingExecutionSpec(userId);
             // Process the document into chunks under one immutable execution spec.
@@ -791,7 +812,8 @@ export class DocumentService {
             }
             buffers.push(buffer);
         }
-        const extracted = await extractDocumentContent(Buffer.concat(buffers, size), source.descriptor.contentType, signal);
+        const sourceBuffer = Buffer.concat(buffers, size);
+        const extracted = await extractDocumentContent(source.descriptor.originalFilename ?? current.filename, sourceBuffer, source.descriptor.contentType, signal);
         let document = {
             ...current,
             content: extracted.content,
@@ -804,6 +826,10 @@ export class DocumentService {
             ...(document.metadata ?? {}),
             processingStatus: 'completed',
             processedAt: Date.now(),
+            contentSha256: documentContentSha256(sourceBuffer),
+            ...(extracted.segments.length > 0
+                ? { segments: extracted.segments }
+                : {}),
         };
         document = withDocumentIndexMetadata(document, embedded, spec);
         await assertSideEffectAllowed?.();
@@ -837,6 +863,31 @@ export class DocumentService {
     async getDocument(documentId, userId) {
         return storageService.getDocument(documentId, userId);
     }
+    /**
+     * Finds an existing live document with identical source bytes in the same
+     * scope (the same session, or both standing). Upload deduplication only —
+     * a match in another scope is not a duplicate because deleting one scope's
+     * copy must never remove another's.
+     */
+    async findDuplicateDocument(userId, sessionId, contentSha256) {
+        const scope = sessionId ?? null;
+        const platform = getPlatformStorageRuntime();
+        const documents = await storageService.getAllDocuments(userId);
+        for (const candidate of documents) {
+            if ((candidate.sessionId ?? null) !== scope)
+                continue;
+            if (candidate.metadata?.contentSha256 !== contentSha256)
+                continue;
+            const reserved = await platform.domains.resourceDeletions.isReserved('document', candidate.id);
+            if (!reserved)
+                return candidate;
+        }
+        return undefined;
+    }
+    /** Documents in a chat's searchable scope: session uploads, selected collections, and standing uploads. */
+    async getDocumentsInScope(userId, sessionId, collectionIds) {
+        return (await storageService.getAllDocuments(userId)).filter(document => this.documentInScope(document, sessionId, collectionIds));
+    }
     async getDocuments(userId, sessionId) {
         const allDocs = await storageService.getAllDocuments(userId);
         if (sessionId) {
@@ -892,6 +943,147 @@ export class DocumentService {
             document.collectionId &&
             collectionIds.includes(document.collectionId));
     }
+    /**
+     * Principals that may retrieve this document's vectors: everyone granted
+     * the document directly plus everyone granted its collection. Grants ride
+     * the ACL table, so revocation applies on the very next query without
+     * re-embedding anything.
+     */
+    async vectorGrantsForDocument(document) {
+        const security = getPersistence(encryptionService).repositories.security;
+        const rows = [
+            ...(await security.grants.listForResource('document', document.id)),
+        ];
+        if (document.collectionId) {
+            rows.push(...(await security.grants.listForResource('knowledge-collection', document.collectionId)));
+        }
+        const unique = new Map();
+        for (const row of rows) {
+            if (row.principal_type !== 'user' && row.principal_type !== 'group') {
+                continue;
+            }
+            unique.set(`${row.principal_type}${row.principal_id}`, {
+                type: row.principal_type,
+                id: row.principal_id,
+            });
+        }
+        return [...unique.values()];
+    }
+    /**
+     * Re-publishes the vector ACL for every document a share change affects.
+     * Called after a grant on a document or knowledge collection is created
+     * or revoked.
+     */
+    async syncShareGrants(resourceType, resourceId, ownerUserId) {
+        const platform = getPlatformStorageRuntime();
+        const documents = resourceType === 'document'
+            ? await storageService
+                .getDocument(resourceId, ownerUserId)
+                .then(document => (document ? [document] : []))
+            : (await storageService.getAllDocuments(ownerUserId)).filter(document => document.collectionId === resourceId);
+        for (const document of documents) {
+            const grants = await this.vectorGrantsForDocument(document);
+            await platform.vectorStore.replaceResourceGrants({
+                actor: { userId: ownerUserId },
+                namespace: DOCUMENT_VECTOR_NAMESPACE,
+                resourceIds: [document.id],
+                grants,
+            });
+        }
+    }
+    /**
+     * Documents living in knowledge collections shared with the actor. The
+     * grant rows themselves are the authorization: they are resolved fresh
+     * (including group membership) on every call.
+     */
+    async loadSharedScopeDocuments(userId, collectionIds) {
+        const grants = await listGrantsForActor({ userId });
+        const shared = [];
+        const seenCollections = new Set();
+        const seenDocuments = new Set();
+        for (const grant of grants) {
+            if (grant.resource_type !== 'knowledge-collection')
+                continue;
+            const collectionId = grant.resource_id;
+            if (seenCollections.has(collectionId))
+                continue;
+            seenCollections.add(collectionId);
+            if (grant.owner_user_id === userId)
+                continue;
+            if (collectionIds &&
+                collectionIds.length > 0 &&
+                !collectionIds.includes(collectionId)) {
+                continue;
+            }
+            const documents = (await storageService.getAllDocuments(grant.owner_user_id)).filter(document => document.collectionId === collectionId);
+            for (const document of documents) {
+                if (seenDocuments.has(document.id))
+                    continue;
+                seenDocuments.add(document.id);
+                shared.push({ document, ownerUserId: grant.owner_user_id });
+            }
+        }
+        return shared;
+    }
+    /**
+     * Loads a document the actor may read: their own, or one reachable
+     * through a document or collection grant. Shared results are read-only.
+     */
+    async getDocumentShared(documentId, actor) {
+        const own = await storageService.getDocument(documentId, actor.userId);
+        if (own)
+            return { document: own, ownerUserId: actor.userId };
+        const ownerUserId = await getPersistence(encryptionService).repositories.resources.archive.ownerOf('document', documentId);
+        if (!ownerUserId || ownerUserId === actor.userId)
+            return undefined;
+        const document = await storageService.getDocument(documentId, ownerUserId);
+        if (!document)
+            return undefined;
+        let shared = await sharedMetaFor(actor, 'document', documentId, ownerUserId);
+        if (!shared && document.collectionId) {
+            shared = await sharedMetaFor(actor, 'knowledge-collection', document.collectionId, ownerUserId);
+        }
+        if (!shared)
+            return undefined;
+        return { document, ownerUserId, shared };
+    }
+    /** Documents shared with the actor through collection grants. */
+    async listSharedDocuments(actor) {
+        const entries = await this.loadSharedScopeDocuments(actor.userId);
+        const result = [];
+        for (const { document, ownerUserId } of entries) {
+            if (!document.collectionId)
+                continue;
+            const meta = await sharedMetaFor(actor, 'knowledge-collection', document.collectionId, ownerUserId);
+            if (meta)
+                result.push({ document, shared: meta });
+        }
+        return result;
+    }
+    /**
+     * The actor's own collections followed by collections shared with them.
+     */
+    async listCollectionsWithShared(actor) {
+        const own = await storageService.getKnowledgeCollections(actor.userId);
+        const sharedIds = await grantedResourceIdsFor(actor, 'knowledge-collection', new Set(own.map(collection => collection.id)));
+        const shared = [];
+        for (const collectionId of sharedIds) {
+            const collection = await storageService.getKnowledgeCollectionById(collectionId);
+            if (!collection || collection.ownerUserId === actor.userId)
+                continue;
+            const meta = await sharedMetaFor(actor, 'knowledge-collection', collectionId, collection.ownerUserId);
+            if (!meta)
+                continue;
+            shared.push({
+                id: collection.id,
+                name: collection.name,
+                createdAt: collection.createdAt,
+                updatedAt: collection.updatedAt,
+                shared: meta,
+            });
+        }
+        return [...own, ...shared];
+    }
     async indexDocumentChunks(document, chunks, userId, spec, attemptCount = 1, signal, assertSideEffectAllowed) {
         const executionSpec = spec ?? (await this.captureEmbeddingExecutionSpec(userId));
         const embedded = embeddedChunks(chunks);
@@ -924,6 +1116,7 @@ export class DocumentService {
             mutated = true;
             await assertSideEffectAllowed?.();
             await this.assertDocumentIndexAuthoritative(document.id, userId, expectedRevision, expectedSource);
+            const shareGrants = await this.vectorGrantsForDocument(document);
             for (let offset = 0; offset < embedded.length; offset += MAX_VECTOR_RECORDS_PER_UPSERT) {
                 await assertSideEffectAllowed?.();
                 await this.assertDocumentIndexAuthoritative(document.id, userId, expectedRevision, expectedSource);
@@ -947,6 +1140,7 @@ export class DocumentService {
                                 ? { collectionId: document.collectionId }
                                 : {}),
                         },
+                        ...(shareGrants.length > 0 ? { grants: shareGrants } : {}),
                     })),
                 });
                 await assertSideEffectAllowed?.();
@@ -992,9 +1186,19 @@ export class DocumentService {
                 return this.keywordSearchDocuments(query, userId, sessionId, limit, collectionIds, signal);
             }
             const documents = (await storageService.getAllDocuments(userId)).filter(document => this.documentInScope(document, sessionId, collectionIds));
+            // Shared-collection documents join the corpus read-only: their chunks
+            // feed the lexical ranking directly, and their vector hits pass the
+            // store's ACL predicate through the grants published at share time.
+            const sharedDocuments = await this.loadSharedScopeDocuments(userId, collectionIds);
             const platform = getPlatformStorageRuntime();
             const chunksById = new Map();
-            const unembedded = [];
+            for (const { document } of sharedDocuments) {
+                throwIfChatGenerationCancelled(signal);
+                const documentChunks = await this.loadDocumentChunks(document.id);
+                for (const chunk of documentChunks) {
+                    chunksById.set(chunk.id, { chunk, document });
+                }
+            }
             for (const document of documents) {
                 throwIfChatGenerationCancelled(signal);
                 let searchableDocument = document;
@@ -1026,7 +1230,6 @@ export class DocumentService {
                 // and deletion cleanup through the shared document resource lease.
                 // PostgreSQL/team reads never mutate PGVector.
                 const publishedIndex = readDocumentIndexMetadata(searchableDocument);
-                const canProveCurrentEmbeddingModel = documentIndexMatchesSpec(publishedIndex, executionSpec);
                 const localIndexProbe = platform.dialect === 'sqlite'
                     ? exactLocalDocumentIndexProbe(searchableDocument, documentChunks, userId, executionSpec)
                     : undefined;
@@ -1069,24 +1272,20 @@ export class DocumentService {
                 for (const chunk of documentChunks) {
                     // PostgreSQL deliberately keeps embeddings only in PGVector, so a
                     // relational chunk without an `embedding` property can still be the
-                    // authoritative content for a vector hit. Always make every current
-                    // relational chunk available for hit hydration. Metadata tells us
-                    // whether an absent inline embedding means "stored externally" or
-                    // whether this chunk genuinely needs keyword fallback.
+                    // authoritative content for a vector hit. Every current relational
+                    // chunk joins the candidate set: vector hits hydrate from it and
+                    // the BM25 side of the hybrid ranking scores it, so chunks without
+                    // a current embedding stay reachable.
                     chunksById.set(chunk.id, {
                         chunk,
                         document: searchableDocument,
                     });
-                    const indexedForSemanticQuery = canProveCurrentEmbeddingModel &&
-                        publishedIndex !== undefined &&
-                        publishedIndex.dimensions !== null &&
-                        (platform.dialect === 'postgres' || localIndexReady);
-                    if (!indexedForSemanticQuery) {
-                        unembedded.push({ chunk, document: searchableDocument });
-                    }
                 }
             }
-            const resourceIds = documents.map(document => document.id);
+            const resourceIds = [
+                ...documents.map(document => document.id),
+                ...sharedDocuments.map(entry => entry.document.id),
+            ];
             const hits = [];
             for (let offset = 0; offset < resourceIds.length; offset += MAX_VECTOR_RESOURCE_FILTERS) {
                 throwIfChatGenerationCancelled(signal);
@@ -1106,37 +1305,39 @@ export class DocumentService {
             hits.sort((left, right) => right.score - left.score ||
                 left.ownerUserId.localeCompare(right.ownerUserId) ||
                 left.id.localeCompare(right.id));
-            const top = [];
-            const selectedChunkIds = new Set();
+            // Hybrid assembly: fuse the vector ranking with an in-process BM25
+            // ranking over every in-scope chunk. The lexical side keeps chunks
+            // without current embeddings reachable and lets an exact term match
+            // outrank a semantically weak neighbor; fusion means neither ranking
+            // silently dominates the other.
+            const semanticRanked = [];
+            const semanticScores = new Map();
             for (const hit of hits) {
-                if (top.length >= limit)
-                    break;
                 const match = chunksById.get(hit.id);
                 if (!match ||
                     match.document.id !== hit.resourceId ||
                     sourceRevision(match.chunk) !== hit.sourceRevision ||
-                    selectedChunkIds.has(match.chunk.id)) {
+                    semanticScores.has(match.chunk.id)) {
                     continue;
                 }
-                top.push({
-                    ...match.chunk,
-                    filename: match.document.filename,
-                });
-                selectedChunkIds.add(match.chunk.id);
+                semanticRanked.push(match.chunk.id);
+                semanticScores.set(match.chunk.id, hit.score);
             }
-            if (top.length < limit && unembedded.length > 0) {
-                const scored = this.scoreChunksByKeywords(query, unembedded);
-                for (const entry of scored) {
-                    if (top.length >= limit)
-                        break;
-                    if (selectedChunkIds.has(entry.chunk.id))
-                        continue;
-                    top.push({
-                        ...entry.chunk,
-                        filename: entry.document.filename,
-                    });
-                    selectedChunkIds.add(entry.chunk.id);
-                }
+            const lexicalScored = this.scoreChunksByKeywords(query, [
+                ...chunksById.values(),
+            ]);
+            const fused = reciprocalRankFusion([
+                semanticRanked,
+                lexicalScored.map(entry => entry.chunk.id),
+            ]);
+            const top = [];
+            for (const { id, score } of fused) {
+                if (top.length >= limit)
+                    break;
+                const match = chunksById.get(id);
+                if (!match)
+                    continue;
+                top.push(this.hydrateRetrievedChunk(match.chunk, match.document, score));
             }
             if (top.length > 0)
                 return top;
@@ -1151,25 +1352,13 @@ export class DocumentService {
             return this.keywordSearchDocuments(query, userId, sessionId, limit, collectionIds, signal);
         }
     }
-    /** Term-frequency scores for a set of chunks, best first. */
+    /** BM25 scores for a set of chunks, best first. */
     scoreChunksByKeywords(query, entries) {
-        const searchTerms = query
-            .toLowerCase()
-            .split(/\s+/)
-            .filter(term => term.length > 2);
-        const scored = [];
-        for (const { chunk, document } of entries) {
-            const chunkText = chunk.content.toLowerCase();
-            let score = 0;
-            for (const term of searchTerms) {
-                // Escape special regex characters to prevent injection
-                const escapedTerm = term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-                score += (chunkText.match(new RegExp(escapedTerm, 'gi')) || []).length;
-            }
-            if (score > 0)
-                scored.push({ chunk, score, document });
-        }
-        return scored.sort((a, b) => b.score - a.score);
+        const byId = new Map(entries.map(entry => [entry.chunk.id, entry]));
+        return scoreCandidatesBm25(query, entries.map(entry => ({ id: entry.chunk.id, text: entry.chunk.content })), { requireQueryWordMatch: true }).map(({ id, score }) => {
+            const entry = byId.get(id);
+            return { chunk: entry.chunk, score, document: entry.document };
+        });
     }
     async keywordSearchDocuments(query, userId, sessionId, limit = 5, collectionIds, signal) {
         const candidates = [];
@@ -1184,12 +1373,28 @@ export class DocumentService {
                 candidates.push({ chunk, document });
             }
         }
+        for (const { document } of await this.loadSharedScopeDocuments(userId, collectionIds)) {
+            throwIfChatGenerationCancelled(signal);
+            const documentChunks = await this.loadDocumentChunks(document.id);
+            for (const chunk of documentChunks) {
+                candidates.push({ chunk, document });
+            }
+        }
         throwIfChatGenerationCancelled(signal);
         const results = this.scoreChunksByKeywords(query, candidates);
-        return results.slice(0, limit).map(result => ({
-            ...result.chunk,
-            filename: result.document.filename, // Add filename for context
-        }));
+        return results
+            .slice(0, limit)
+            .map(result => this.hydrateRetrievedChunk(result.chunk, result.document, result.score));
+    }
+    /** Attaches filename, score, and source location to a retrieval result. */
+    hydrateRetrievedChunk(chunk, document, score) {
+        const location = resolveSegmentLabel(readDocumentSegments(document.metadata), chunk.startChar, chunk.endChar);
+        return {
+            ...chunk,
+            filename: document.filename,
+            ...(score !== undefined ? { score } : {}),
+            ...(location ? { location } : {}),
+        };
     }
     // Get relevant context for RAG
     async getRelevantContext(query, userId, sessionId) {