@opengeni/documents 0.5.38 → 0.5.41

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import type { Settings } from "@opengeni/config";
-import type { AddDocumentRequest, CreateDocumentBaseRequest, Document, DocumentAuthorityKind, DocumentBase, DocumentCurationStatus, DocumentSearchMode, DocumentSearchResult, DocumentStatus, DocumentVisibility, FileAsset, KnowledgeBrowseResponse, KnowledgeRecord, KnowledgeSearchResponse, KnowledgeSourceKind, ListIndexedDocumentsResponse } from "@opengeni/contracts";
+import { type AddDocumentRequest, CreateDocumentBaseRequest, Document, DocumentAuthorityKind, DocumentBase, DocumentCurationStatus, DocumentSearchMode, DocumentSearchResult, DocumentStatus, DocumentVisibility, FileAsset, KnowledgeBrowseResponse, KnowledgeRecord, KnowledgeSearchResponse, KnowledgeSourceKind, ListIndexedDocumentsResponse } from "@opengeni/contracts";
 import { type Database } from "@opengeni/db";
 import type { ObjectStorage } from "@opengeni/storage";
 export { projectKnowledgeRecord } from "./knowledge-projection.js";
@@ -327,6 +327,14 @@ export declare function decodeDocumentIndexCheckpoint(value: string, scope: {
     initiatingSubjectId: string;
 }): bigint;
 export declare function getDocument(db: Database, workspaceId: string, documentId: string, access?: DocumentAccessFilter): Promise<Document | null>;
+/**
+ * Internal ingestion-only read used after the worker has independently resolved
+ * and fenced the immutable document authority tuple. It deliberately does not
+ * apply provider retrieval authorization because a new Drive document must be
+ * indexed before its first ACL evidence can be attached. User, API, MCP, and
+ * agent reads must use getDocument/effective retrieval instead.
+ */
+export declare function getDocumentForIndexing(db: Database, workspaceId: string, documentId: string): Promise<Document | null>;
 export declare function queueDocumentForReindex(db: Database, workspaceId: string, documentId: string, access?: DocumentAccessFilter, organizationAuthorityGranted?: boolean): Promise<Document>;
 export declare function indexDocumentNow(db: Database, objectStorage: ObjectStorage, workspaceId: string, documentId: string, services?: DocumentServices, hooks?: DocumentIndexHooks, access?: DocumentAccessFilter): Promise<Document>;
 export declare function searchDocuments(db: Database, input: DocumentSearchInput, services?: Pick<DocumentServices, "embedder">): Promise<DocumentSearchResult[]>;

package/dist/index.js

@@ -1,6 +1,9 @@
 // src/index.ts
 import {
-  requireFile,
+  KnowledgeProviderCitation
+} from "@opengeni/contracts";
+import {
+  getFilesForSubject,
   rlsContextForWorkspace,
   setSubjectRlsContext,
   withRlsContext,
@@ -715,7 +718,18 @@ async function addDocumentToBase(db, input) {
       if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
       const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
       if (!base) throw new Error(`Document base not found: ${input.baseId}`);
-      const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
+      const initiatingSubjectId = cleanString(input.initiatingSubjectId ?? null);
+      const createdBy = cleanString(input.createdBy ?? null);
+      if (initiatingSubjectId && createdBy && initiatingSubjectId !== createdBy) {
+        throw new Error("document file authority must match the exact initiating subject");
+      }
+      const fileAuthoritySubjectId = initiatingSubjectId ?? createdBy ?? null;
+      const file = await requireReadyFile(scopedDb, {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        subjectId: fileAuthoritySubjectId,
+        fileId: input.fileId
+      });
       const knowledgeSourceIdentity = cleanString(input.knowledgeSourceIdentity ?? null);
       if (knowledgeSourceIdentity && knowledgeSourceIdentity.length > 512) {
         throw new Error("knowledge source document identity exceeds 512 characters");
@@ -785,7 +799,7 @@ async function addDocumentToBase(db, input) {
         authoritySubjectId: authority.subjectId,
         visibility: authority.kind === "personal" ? "private" : "workspace",
         agentAccess: input.agentAccess ?? true,
-        createdBy: input.createdBy ?? null,
+        createdBy: fileAuthoritySubjectId,
         curationStatus: input.curationStatus ?? "none",
         updatedAt: now
       }).returning();
@@ -1009,6 +1023,14 @@ async function getDocument(db, workspaceId, documentId, access) {
     return row ? mapDocument(row) : null;
   });
 }
+async function getDocumentForIndexing(db, workspaceId, documentId) {
+  return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
+    const [row] = await scopedDb.select().from(schema.documents).where(
+      and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId))
+    ).limit(1);
+    return row ? mapDocument(row) : null;
+  });
+}
 async function queueDocumentForReindex(db, workspaceId, documentId, access, organizationAuthorityGranted) {
   return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
     const [document] = await scopedDb.select({ authorityKind: schema.documents.authorityKind }).from(schema.documents).where(
@@ -1051,7 +1073,12 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
   );
   if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
   let document = loadedDocument;
-  const file = await requireReadyFile(db, workspaceId, document.fileId);
+  const file = await requireReadyFile(db, {
+    accountId: document.accountId,
+    workspaceId,
+    subjectId: cleanString(document.createdBy) ?? null,
+    fileId: document.fileId
+  });
   await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
     await scopedDb.update(schema.documents).set({
       status: "indexing",
@@ -1157,9 +1184,7 @@ async function indexDocumentNow(db, objectStorage, workspaceId, documentId, serv
     if (!failed) throw error;
     return mapDocument(failed);
   }
-  const updated = await getDocument(db, workspaceId, documentId, {
-    viewerSubjectId: document.authoritySubjectId
-  });
+  const updated = await getDocumentForIndexing(db, workspaceId, documentId);
   if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
   return updated;
 }
@@ -1325,7 +1350,11 @@ async function searchEffectiveKnowledge(db, input, services = createDocumentServ
     input.accountId,
     input.workspaceId,
     access,
-    async (scopedDb) => await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
+    async (scopedDb) => await scopedDb.select({
+      chunk: schema.documentChunks,
+      document: schema.documents,
+      citation: googleDriveCitationProjection(input.workspaceId, access)
+    }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
       and(
         eq(schema.documents.accountId, input.accountId),
         eq(schema.documentChunks.accountId, input.accountId),
@@ -1345,7 +1374,7 @@ async function searchEffectiveKnowledge(db, input, services = createDocumentServ
       if (!row) return [];
       return [
         {
-          record: knowledgeChunkRecord(row.document, row.chunk),
+          record: knowledgeChunkRecord(row.document, row.chunk, row.citation),
           retrieval: {
             score: rankedResult.score,
             matchType: rankedResult.matchType,
@@ -1368,7 +1397,10 @@ async function getEffectiveKnowledgeRecord(db, input) {
     access,
     async (scopedDb) => {
       if (target.kind === "document") {
-        const [document] = await scopedDb.select().from(schema.documents).where(
+        const [row2] = await scopedDb.select({
+          document: schema.documents,
+          citation: googleDriveCitationProjection(input.workspaceId, access)
+        }).from(schema.documents).where(
           and(
             eq(schema.documents.accountId, input.accountId),
             eq(schema.documents.id, target.id),
@@ -1376,9 +1408,13 @@ async function getEffectiveKnowledgeRecord(db, input) {
             ...documentAccessConditions(input.workspaceId, access)
           )
         ).limit(1);
-        return document ? knowledgeDocumentRecord(document) : null;
+        return row2 ? knowledgeDocumentRecord(row2.document, row2.citation) : null;
       }
-      const [row] = await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
+      const [row] = await scopedDb.select({
+        chunk: schema.documentChunks,
+        document: schema.documents,
+        citation: googleDriveCitationProjection(input.workspaceId, access)
+      }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
         and(
           eq(schema.documents.accountId, input.accountId),
           eq(schema.documentChunks.accountId, input.accountId),
@@ -1387,7 +1423,7 @@ async function getEffectiveKnowledgeRecord(db, input) {
           ...documentAccessConditions(input.workspaceId, access)
         )
       ).limit(1);
-      return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
+      return row ? knowledgeChunkRecord(row.document, row.chunk, row.citation) : null;
     }
   );
 }
@@ -1438,7 +1474,11 @@ async function browseEffectiveKnowledge(db, input) {
           )
         ).limit(1);
         if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
-        const rows2 = await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
+        const rows2 = await scopedDb.select({
+          chunk: schema.documentChunks,
+          document: schema.documents,
+          citation: googleDriveCitationProjection(input.workspaceId, access)
+        }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
           and(
             eq(schema.documentChunks.accountId, input.accountId),
             eq(schema.documentChunks.documentId, parent.id),
@@ -1451,7 +1491,7 @@ async function browseEffectiveKnowledge(db, input) {
         const page2 = rows2.slice(0, limit);
         const last2 = page2.at(-1)?.chunk.chunkIndex;
         return {
-          records: page2.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
+          records: page2.map((row) => knowledgeChunkRecord(row.document, row.chunk, row.citation)),
           nextCursor: hasMore2 && last2 !== void 0 ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last2 + 1)) : null,
           hasMore: hasMore2
         };
@@ -1466,12 +1506,15 @@ async function browseEffectiveKnowledge(db, input) {
       if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
       if (sourceKinds.length > 0)
         conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
-      const rows = await scopedDb.select().from(schema.documents).where(and(...conditions)).orderBy(asc(schema.documents.indexSequence)).limit(limit + 1);
+      const rows = await scopedDb.select({
+        document: schema.documents,
+        citation: googleDriveCitationProjection(input.workspaceId, access)
+      }).from(schema.documents).where(and(...conditions)).orderBy(asc(schema.documents.indexSequence)).limit(limit + 1);
       const hasMore = rows.length > limit;
       const page = rows.slice(0, limit);
-      const last = page.at(-1)?.indexSequence;
+      const last = page.at(-1)?.document.indexSequence;
       return {
-        records: page.map(knowledgeDocumentRecord),
+        records: page.map((row) => knowledgeDocumentRecord(row.document, row.citation)),
         nextCursor: hasMore && last !== void 0 && last !== null ? encodeKnowledgeBrowseCursor(cursorScope, last) : null,
         hasMore
       };
@@ -1519,7 +1562,7 @@ function parseKnowledgeRecordId(value) {
   if (!match) throw new Error("invalid knowledge record id");
   return { kind: match[1], id: match[2].toLowerCase() };
 }
-function knowledgeDocumentRecord(document) {
+function knowledgeDocumentRecord(document, citation = null) {
   if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
   const projected = projectKnowledgeRecord({
     title: document.title,
@@ -1537,7 +1580,8 @@ function knowledgeDocumentRecord(document) {
     authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
     provenance: {
       source: projected.source,
-      indexedAt: document.indexedAt.toISOString()
+      indexedAt: document.indexedAt.toISOString(),
+      citation: parseKnowledgeProviderCitation(citation)
     },
     lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
     quality: knowledgeQuality(document),
@@ -1545,7 +1589,7 @@ function knowledgeDocumentRecord(document) {
     projection: projected.projection
   };
 }
-function knowledgeChunkRecord(document, chunk) {
+function knowledgeChunkRecord(document, chunk, citation = null) {
   if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
   const projected = projectKnowledgeRecord({
     title: document.title,
@@ -1563,7 +1607,8 @@ function knowledgeChunkRecord(document, chunk) {
     authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
     provenance: {
       source: projected.source,
-      indexedAt: document.indexedAt.toISOString()
+      indexedAt: document.indexedAt.toISOString(),
+      citation: parseKnowledgeProviderCitation(citation)
     },
     lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
     quality: knowledgeQuality(document),
@@ -1629,6 +1674,7 @@ async function vectorSearchDocuments(db, input, limit, services) {
       authorityKind: schema.documents.authorityKind,
       authorityWorkspaceId: schema.documents.authorityWorkspaceId,
       authoritySubjectId: schema.documents.authoritySubjectId,
+      citation: googleDriveCitationProjection(input.workspaceId, input.access),
       distance
     }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(and(...documentSearchConditions(input, services.embedder.model))).orderBy(distance).limit(limit)
   );
@@ -1667,6 +1713,7 @@ async function keywordSearchDocuments(db, input, limit) {
       authorityKind: schema.documents.authorityKind,
       authorityWorkspaceId: schema.documents.authorityWorkspaceId,
       authoritySubjectId: schema.documents.authoritySubjectId,
+      citation: googleDriveCitationProjection(input.workspaceId, input.access),
       rank
     }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
       and(
@@ -1708,7 +1755,8 @@ async function getDocumentChunk(db, accountId, workspaceId, chunkId, access) {
       aclTags: schema.documents.aclTags,
       authorityKind: schema.documents.authorityKind,
       authorityWorkspaceId: schema.documents.authorityWorkspaceId,
-      authoritySubjectId: schema.documents.authoritySubjectId
+      authoritySubjectId: schema.documents.authoritySubjectId,
+      citation: googleDriveCitationProjection(workspaceId, access)
     }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
       and(
         eq(schema.documents.accountId, accountId),
@@ -1781,10 +1829,17 @@ function documentAccessConditions(workspaceId, access) {
     eq(schema.documents.authoritySubjectId, viewer)
   ) : void 0;
   const authority = viewer ? or(organization, workspace, personal) ?? organization : or(organization, workspace) ?? organization;
+  const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
+  const providerAuthorization = sql`google_drive_file_authorized(
+    ${schema.documents.accountId},
+    ${workspaceId}::uuid,
+    ${viewerSql},
+    ${schema.documents.fileId}
+  )`;
   if (access?.agentOnly) {
-    return [eq(schema.documents.agentAccess, true), authority];
+    return [eq(schema.documents.agentAccess, true), authority, providerAuthorization];
   }
-  return [authority];
+  return [authority, providerAuthorization];
 }
 function documentMatchesAccess(document, workspaceId, access) {
   if (access?.agentOnly) {
@@ -1863,9 +1918,24 @@ function mapSearchRowBase(row) {
     aclTags: cleanStringArray(row.aclTags),
     authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
     authorityWorkspaceId: row.authorityWorkspaceId,
-    authoritySubjectId: row.authoritySubjectId
+    authoritySubjectId: row.authoritySubjectId,
+    citation: parseKnowledgeProviderCitation(row.citation)
   };
 }
+function googleDriveCitationProjection(workspaceId, access) {
+  const viewer = cleanString(access?.viewerSubjectId ?? null);
+  const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
+  return sql`google_drive_document_citation(
+    ${schema.documents.accountId},
+    ${workspaceId}::uuid,
+    ${viewerSql},
+    ${schema.documents.id},
+    ${schema.documents.fileId}
+  )`;
+}
+function parseKnowledgeProviderCitation(value) {
+  return value === null || value === void 0 ? null : KnowledgeProviderCitation.parse(value);
+}
 function mergeDocumentSearchRows(rows, mode) {
   const byChunk = /* @__PURE__ */ new Map();
   for (const row of rows) {
@@ -1952,10 +2022,16 @@ function deterministicEmbedding(text, dimensions = DEFAULT_DOCUMENT_EMBEDDING_DI
   const norm = Math.hypot(...values) || 1;
   return values.map((value) => Number((value / norm).toFixed(6)));
 }
-async function requireReadyFile(db, workspaceId, fileId) {
-  const file = await requireFile(db, workspaceId, fileId);
+async function requireReadyFile(db, input) {
+  const [file] = await getFilesForSubject(db, {
+    accountId: input.accountId,
+    workspaceId: input.workspaceId,
+    subjectId: input.subjectId,
+    fileIds: [input.fileId]
+  });
+  if (!file) throw new Error(`File not found: ${input.fileId}`);
   if (file.status !== "ready") {
-    throw new Error(`File ${fileId} is ${file.status}`);
+    throw new Error(`File ${input.fileId} is ${file.status}`);
   }
   return file;
 }
@@ -2175,6 +2251,7 @@ export {
   getDocument,
   getDocumentBase,
   getDocumentChunk,
+  getDocumentForIndexing,
   getDocumentInventory,
   getEffectiveKnowledgeRecord,
   heuristicCuration,