@opengeni/documents 0.5.38 → 0.5.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/documents",
-  "version": "0.5.38",
+  "version": "0.5.41",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -40,10 +40,10 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "^1.5.3",
-    "@opengeni/config": "^0.16.1",
-    "@opengeni/contracts": "^0.50.0",
-    "@opengeni/db": "^0.36.0",
-    "@opengeni/storage": "^0.2.93",
+    "@opengeni/config": "^0.16.4",
+    "@opengeni/contracts": "^1.0.1",
+    "@opengeni/db": "^1.0.1",
+    "@opengeni/storage": "^0.2.96",
     "drizzle-orm": "^0.45.2",
     "openai": "6.47.0"
   }

package/src/index.ts

@@ -1,6 +1,8 @@
 import type { Settings } from "@opengeni/config";
-import type {
-  AddDocumentRequest,
+import {
+  KnowledgeProviderCitation,
+  type KnowledgeProviderCitation as KnowledgeProviderCitationValue,
+  type AddDocumentRequest,
   CreateDocumentBaseRequest,
   Document,
   DocumentAuthorityKind,
@@ -20,7 +22,7 @@ import type {
   ListIndexedDocumentsResponse,
 } from "@opengeni/contracts";
 import {
-  requireFile,
+  getFilesForSubject,
   rlsContextForWorkspace,
   setSubjectRlsContext,
   withRlsContext,
@@ -967,7 +969,18 @@ export async function addDocumentToBase(
       if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
       const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
       if (!base) throw new Error(`Document base not found: ${input.baseId}`);
-      const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+      const initiatingSubjectId = cleanString(input.initiatingSubjectId ?? null);
+      const createdBy = cleanString(input.createdBy ?? null);
+      if (initiatingSubjectId && createdBy && initiatingSubjectId !== createdBy) {
+        throw new Error("document file authority must match the exact initiating subject");
+      }
+      const fileAuthoritySubjectId = initiatingSubjectId ?? createdBy ?? null;
+      const file = await requireReadyFile(scopedDb, {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        subjectId: fileAuthoritySubjectId,
+        fileId: input.fileId,
+      });
       const knowledgeSourceIdentity = cleanString(input.knowledgeSourceIdentity ?? null);
       if (knowledgeSourceIdentity && knowledgeSourceIdentity.length > 512) {
         throw new Error("knowledge source document identity exceeds 512 characters");
@@ -1054,7 +1067,7 @@ export async function addDocumentToBase(
           authoritySubjectId: authority.subjectId,
           visibility: authority.kind === "personal" ? "private" : "workspace",
           agentAccess: input.agentAccess ?? true,
-          createdBy: input.createdBy ?? null,
+          createdBy: fileAuthoritySubjectId,
           curationStatus: input.curationStatus ?? "none",
           updatedAt: now,
         })
@@ -1403,6 +1416,30 @@ export async function getDocument(
   });
 }
 
+/**
+ * Internal ingestion-only read used after the worker has independently resolved
+ * and fenced the immutable document authority tuple. It deliberately does not
+ * apply provider retrieval authorization because a new Drive document must be
+ * indexed before its first ACL evidence can be attached. User, API, MCP, and
+ * agent reads must use getDocument/effective retrieval instead.
+ */
+export async function getDocumentForIndexing(
+  db: Database,
+  workspaceId: string,
+  documentId: string,
+): Promise<Document | null> {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const [row] = await scopedDb
+      .select()
+      .from(schema.documents)
+      .where(
+        and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
+      )
+      .limit(1);
+    return row ? mapDocument(row) : null;
+  });
+}
+
 export async function queueDocumentForReindex(
   db: Database,
   workspaceId: string,
@@ -1477,7 +1514,12 @@ export async function indexDocumentNow(
   );
   if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
   let document: DocumentRow = loadedDocument;
-  const file = await requireReadyFile(db, workspaceId, document.fileId);
+  const file = await requireReadyFile(db, {
+    accountId: document.accountId,
+    workspaceId,
+    subjectId: cleanString(document.createdBy) ?? null,
+    fileId: document.fileId,
+  });
   await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
     await scopedDb
       .update(schema.documents)
@@ -1604,9 +1646,7 @@ export async function indexDocumentNow(
   // Internal indexing must be able to return a private document to the caller
   // that created/queued it. Public reads remain fail-closed when no subject is
   // supplied; the creator subject is the document's frozen access principal.
-  const updated = await getDocument(db, workspaceId, documentId, {
-    viewerSubjectId: document.authoritySubjectId,
-  });
+  const updated = await getDocumentForIndexing(db, workspaceId, documentId);
   if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
   return updated;
 }
@@ -1839,7 +1879,11 @@ export async function searchEffectiveKnowledge(
     access,
     async (scopedDb) =>
       await scopedDb
-        .select({ chunk: schema.documentChunks, document: schema.documents })
+        .select({
+          chunk: schema.documentChunks,
+          document: schema.documents,
+          citation: googleDriveCitationProjection(input.workspaceId, access),
+        })
         .from(schema.documentChunks)
         .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
         .where(
@@ -1862,7 +1906,7 @@ export async function searchEffectiveKnowledge(
       if (!row) return [];
       return [
         {
-          record: knowledgeChunkRecord(row.document, row.chunk),
+          record: knowledgeChunkRecord(row.document, row.chunk, row.citation),
           retrieval: {
             score: rankedResult.score,
             matchType: rankedResult.matchType,
@@ -1895,8 +1939,11 @@ export async function getEffectiveKnowledgeRecord(
     access,
     async (scopedDb) => {
       if (target.kind === "document") {
-        const [document] = await scopedDb
-          .select()
+        const [row] = await scopedDb
+          .select({
+            document: schema.documents,
+            citation: googleDriveCitationProjection(input.workspaceId, access),
+          })
           .from(schema.documents)
           .where(
             and(
@@ -1907,10 +1954,14 @@ export async function getEffectiveKnowledgeRecord(
             ),
           )
           .limit(1);
-        return document ? knowledgeDocumentRecord(document) : null;
+        return row ? knowledgeDocumentRecord(row.document, row.citation) : null;
       }
       const [row] = await scopedDb
-        .select({ chunk: schema.documentChunks, document: schema.documents })
+        .select({
+          chunk: schema.documentChunks,
+          document: schema.documents,
+          citation: googleDriveCitationProjection(input.workspaceId, access),
+        })
         .from(schema.documentChunks)
         .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
         .where(
@@ -1923,7 +1974,7 @@ export async function getEffectiveKnowledgeRecord(
           ),
         )
         .limit(1);
-      return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
+      return row ? knowledgeChunkRecord(row.document, row.chunk, row.citation) : null;
     },
   );
 }
@@ -1988,7 +2039,11 @@ export async function browseEffectiveKnowledge(
           .limit(1);
         if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
         const rows = await scopedDb
-          .select({ chunk: schema.documentChunks, document: schema.documents })
+          .select({
+            chunk: schema.documentChunks,
+            document: schema.documents,
+            citation: googleDriveCitationProjection(input.workspaceId, access),
+          })
           .from(schema.documentChunks)
           .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
           .where(
@@ -2006,7 +2061,7 @@ export async function browseEffectiveKnowledge(
         const page = rows.slice(0, limit);
         const last = page.at(-1)?.chunk.chunkIndex;
         return {
-          records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
+          records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk, row.citation)),
           nextCursor:
             hasMore && last !== undefined
               ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last + 1))
@@ -2026,16 +2081,19 @@ export async function browseEffectiveKnowledge(
       if (sourceKinds.length > 0)
         conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
       const rows = await scopedDb
-        .select()
+        .select({
+          document: schema.documents,
+          citation: googleDriveCitationProjection(input.workspaceId, access),
+        })
         .from(schema.documents)
         .where(and(...conditions))
         .orderBy(asc(schema.documents.indexSequence))
         .limit(limit + 1);
       const hasMore = rows.length > limit;
       const page = rows.slice(0, limit);
-      const last = page.at(-1)?.indexSequence;
+      const last = page.at(-1)?.document.indexSequence;
       return {
-        records: page.map(knowledgeDocumentRecord),
+        records: page.map((row) => knowledgeDocumentRecord(row.document, row.citation)),
         nextCursor:
           hasMore && last !== undefined && last !== null
             ? encodeKnowledgeBrowseCursor(cursorScope, last)
@@ -2132,7 +2190,10 @@ function parseKnowledgeRecordId(value: string): {
   return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
 }
 
-function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect): KnowledgeRecord {
+function knowledgeDocumentRecord(
+  document: typeof schema.documents.$inferSelect,
+  citation: unknown = null,
+): KnowledgeRecord {
   if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
   const projected = projectKnowledgeRecord({
     title: document.title,
@@ -2151,6 +2212,7 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
     provenance: {
       source: projected.source,
       indexedAt: document.indexedAt.toISOString(),
+      citation: parseKnowledgeProviderCitation(citation),
     },
     lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
     quality: knowledgeQuality(document),
@@ -2162,6 +2224,7 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
 function knowledgeChunkRecord(
   document: typeof schema.documents.$inferSelect,
   chunk: typeof schema.documentChunks.$inferSelect,
+  citation: unknown = null,
 ): KnowledgeRecord {
   if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
   const projected = projectKnowledgeRecord({
@@ -2181,6 +2244,7 @@ function knowledgeChunkRecord(
     provenance: {
       source: projected.source,
       indexedAt: document.indexedAt.toISOString(),
+      citation: parseKnowledgeProviderCitation(citation),
     },
     lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
     quality: knowledgeQuality(document),
@@ -2259,6 +2323,7 @@ async function vectorSearchDocuments(
           authorityKind: schema.documents.authorityKind,
           authorityWorkspaceId: schema.documents.authorityWorkspaceId,
           authoritySubjectId: schema.documents.authoritySubjectId,
+          citation: googleDriveCitationProjection(input.workspaceId, input.access),
           distance,
         })
         .from(schema.documentChunks)
@@ -2309,6 +2374,7 @@ async function keywordSearchDocuments(
           authorityKind: schema.documents.authorityKind,
           authorityWorkspaceId: schema.documents.authorityWorkspaceId,
           authoritySubjectId: schema.documents.authoritySubjectId,
+          citation: googleDriveCitationProjection(input.workspaceId, input.access),
           rank,
         })
         .from(schema.documentChunks)
@@ -2365,6 +2431,7 @@ export async function getDocumentChunk(
           authorityKind: schema.documents.authorityKind,
           authorityWorkspaceId: schema.documents.authorityWorkspaceId,
           authoritySubjectId: schema.documents.authoritySubjectId,
+          citation: googleDriveCitationProjection(workspaceId, access),
         })
         .from(schema.documentChunks)
         .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
@@ -2492,10 +2559,17 @@ function documentAccessConditions(
   const authority = viewer
     ? (or(organization, workspace, personal) ?? organization)
     : (or(organization, workspace) ?? organization);
+  const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
+  const providerAuthorization = sql`google_drive_file_authorized(
+    ${schema.documents.accountId},
+    ${workspaceId}::uuid,
+    ${viewerSql},
+    ${schema.documents.fileId}
+  )`;
   if (access?.agentOnly) {
-    return [eq(schema.documents.agentAccess, true), authority];
+    return [eq(schema.documents.agentAccess, true), authority, providerAuthorization];
   }
-  return [authority];
+  return [authority, providerAuthorization];
 }
 
 function documentMatchesAccess(
@@ -2606,6 +2680,7 @@ function mapSearchRowBase(row: {
   authorityKind: string;
   authorityWorkspaceId: string | null;
   authoritySubjectId: string | null;
+  citation?: unknown;
 }): SearchRowBase {
   return {
     chunkId: row.chunkId,
@@ -2629,9 +2704,29 @@ function mapSearchRowBase(row: {
     authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
     authorityWorkspaceId: row.authorityWorkspaceId,
     authoritySubjectId: row.authoritySubjectId,
+    citation: parseKnowledgeProviderCitation(row.citation),
   };
 }
 
+function googleDriveCitationProjection(
+  workspaceId: string,
+  access: DocumentAccessFilter | undefined,
+): SQL<unknown> {
+  const viewer = cleanString(access?.viewerSubjectId ?? null);
+  const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
+  return sql`google_drive_document_citation(
+    ${schema.documents.accountId},
+    ${workspaceId}::uuid,
+    ${viewerSql},
+    ${schema.documents.id},
+    ${schema.documents.fileId}
+  )`;
+}
+
+function parseKnowledgeProviderCitation(value: unknown): KnowledgeProviderCitationValue | null {
+  return value === null || value === undefined ? null : KnowledgeProviderCitation.parse(value);
+}
+
 function mergeDocumentSearchRows(
   rows: CombinedSearchRow[],
   mode: DocumentSearchMode,
@@ -2763,12 +2858,22 @@ export function deterministicEmbedding(
 
 async function requireReadyFile(
   db: Database,
-  workspaceId: string,
-  fileId: string,
+  input: {
+    accountId: string;
+    workspaceId: string;
+    subjectId: string | null;
+    fileId: string;
+  },
 ): Promise<FileAsset> {
-  const file = await requireFile(db, workspaceId, fileId);
+  const [file] = await getFilesForSubject(db, {
+    accountId: input.accountId,
+    workspaceId: input.workspaceId,
+    subjectId: input.subjectId,
+    fileIds: [input.fileId],
+  });
+  if (!file) throw new Error(`File not found: ${input.fileId}`);
   if (file.status !== "ready") {
-    throw new Error(`File ${fileId} is ${file.status}`);
+    throw new Error(`File ${input.fileId} is ${file.status}`);
   }
   return file;
 }