@opengeni/documents 0.5.39 → 0.6.9-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,9 +1,13 @@
1
1
  import type { Settings } from "@opengeni/config";
2
- import type {
3
- AddDocumentRequest,
2
+ import {
3
+ KnowledgeProviderCitation,
4
+ type KnowledgeProviderCitation as KnowledgeProviderCitationValue,
5
+ type AddDocumentRequest,
4
6
  CreateDocumentBaseRequest,
5
7
  Document,
6
8
  DocumentAuthorityKind,
9
+ DocumentAuthorityReclassification,
10
+ ListDocumentAuthorityReclassificationsResponse,
7
11
  DocumentBase,
8
12
  DocumentCuration,
9
13
  DocumentCurationStatus,
@@ -15,12 +19,18 @@ import type {
15
19
  IndexedDocumentSummary,
16
20
  KnowledgeBrowseResponse,
17
21
  KnowledgeRecord,
22
+ KnowledgeSearchResult,
18
23
  KnowledgeSearchResponse,
19
24
  KnowledgeSourceKind,
20
25
  ListIndexedDocumentsResponse,
26
+ type ReclassifyDocumentAuthorityRequest,
27
+ type RunDocumentDefaultCollectionBackfillRequest,
28
+ DocumentDefaultCollectionBackfill,
21
29
  } from "@opengeni/contracts";
22
30
  import {
23
- requireFile,
31
+ createPersonalDocumentAuthority,
32
+ getFilesForSubject,
33
+ resolveDocumentOriginalFileForSubject,
24
34
  rlsContextForWorkspace,
25
35
  setSubjectRlsContext,
26
36
  withRlsContext,
@@ -29,11 +39,21 @@ import {
29
39
  type Database,
30
40
  } from "@opengeni/db";
31
41
  import * as schema from "@opengeni/db/schema";
32
- import type { ObjectStorage } from "@opengeni/storage";
33
- import { createHash } from "node:crypto";
34
- import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
42
+ import { retryWhileMissing, type ObjectStorage } from "@opengeni/storage";
43
+ import { createHash, randomUUID } from "node:crypto";
44
+ import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from "drizzle-orm";
35
45
  import type OpenAI from "openai";
36
- import { KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS } from "@opengeni/contracts";
46
+ import {
47
+ KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS,
48
+ KNOWLEDGE_BROWSE_MAX_LIMIT,
49
+ KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES,
50
+ KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES,
51
+ KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
52
+ KNOWLEDGE_SEARCH_MAX_RESULTS,
53
+ KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
54
+ KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
55
+ KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
56
+ } from "@opengeni/contracts";
37
57
  import { projectKnowledgeRecord } from "./knowledge-projection";
38
58
 
39
59
  export { projectKnowledgeRecord } from "./knowledge-projection";
@@ -136,6 +156,15 @@ export type DocumentAccessFilter = {
136
156
  * only when the agent carries the creating subject as its viewer subject.
137
157
  */
138
158
  agentOnly?: boolean | undefined;
159
+ /** Exact attempt whose grant snapshot must be revalidated in the content query. */
160
+ authorizedPersonalAttempt?:
161
+ | {
162
+ accountId: string;
163
+ workspaceId: string;
164
+ sessionId: string;
165
+ attemptId: string;
166
+ }
167
+ | undefined;
139
168
  };
140
169
 
141
170
  export type DocumentAuthority = {
@@ -144,6 +173,35 @@ export type DocumentAuthority = {
144
173
  subjectId: string | null;
145
174
  };
146
175
 
176
+ /** JSON-safe projection of the opaque capability minted by the API access layer. */
177
+ export type DocumentAccountAdminAuthorization = Readonly<{
178
+ authorizationId: string;
179
+ accountId: string;
180
+ actorSubjectId: string;
181
+ permission: "account:admin";
182
+ }>;
183
+
184
+ export type ReclassifyDocumentAuthorityInput = ReclassifyDocumentAuthorityRequest & {
185
+ accountId: string;
186
+ workspaceId: string;
187
+ documentId: string;
188
+ actorSubjectId: string;
189
+ accountAdminAuthorization: DocumentAccountAdminAuthorization | null;
190
+ };
191
+
192
+ export type RunDocumentDefaultCollectionBackfillInput =
193
+ RunDocumentDefaultCollectionBackfillRequest & {
194
+ accountId: string;
195
+ workspaceId: string;
196
+ actorSubjectId: string;
197
+ accountAdminAuthorization: DocumentAccountAdminAuthorization;
198
+ };
199
+
200
+ export type AgentDocumentAuthorityContext = {
201
+ sessionId: string;
202
+ attemptId: string;
203
+ };
204
+
147
205
  export type DocumentInventoryStatusCounts = Record<DocumentStatus, number>;
148
206
  export type DocumentInventorySourceKindCounts = Record<KnowledgeSourceKind, number>;
149
207
  export type DocumentInventoryAuthorityKindCounts = Record<DocumentAuthorityKind, number>;
@@ -194,6 +252,7 @@ export type EffectiveDocumentSearchInput = Omit<DocumentSearchInput, "access"> &
194
252
  initiatingSubjectId: string;
195
253
  /** Agent retrieval additionally enforces documents.agent_access. */
196
254
  surface: "human" | "agent";
255
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
197
256
  };
198
257
 
199
258
  export type ListEffectiveIndexedDocumentsInput = {
@@ -203,6 +262,7 @@ export type ListEffectiveIndexedDocumentsInput = {
203
262
  initiatingSubjectId: string;
204
263
  checkpoint?: string | undefined;
205
264
  limit?: number | undefined;
265
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
206
266
  };
207
267
 
208
268
  export type EffectiveKnowledgeBrowseInput = {
@@ -210,12 +270,15 @@ export type EffectiveKnowledgeBrowseInput = {
210
270
  workspaceId: string;
211
271
  /** Immutable human subject accepted for the logical request/turn. */
212
272
  initiatingSubjectId: string;
273
+ /** Human inspection bypasses agent_access but still uses exact subject/RLS authority. */
274
+ surface?: "human" | "agent" | undefined;
213
275
  /** Omit to browse top-level documents; pass a document record id for chunks. */
214
276
  parentId?: string | undefined;
215
277
  topic?: string | undefined;
216
278
  sourceKinds?: KnowledgeSourceKind[] | undefined;
217
279
  cursor?: string | undefined;
218
280
  limit?: number | undefined;
281
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
219
282
  };
220
283
 
221
284
  export type DocumentIndexHooks = {
@@ -929,6 +992,188 @@ export async function listDocumentBasesEnsuringDefault(
929
992
  return await listDocumentBases(db, input.workspaceId);
930
993
  }
931
994
 
995
+ export async function runDocumentDefaultCollectionBackfill(
996
+ db: Database,
997
+ input: RunDocumentDefaultCollectionBackfillInput,
998
+ ) {
999
+ return await withDocumentAccountRls(
1000
+ db,
1001
+ input.accountId,
1002
+ input.workspaceId,
1003
+ { viewerSubjectId: input.actorSubjectId },
1004
+ async (scopedDb) => {
1005
+ const command = {
1006
+ accountId: input.accountId,
1007
+ workspaceId: input.workspaceId,
1008
+ actorSubjectId: input.actorSubjectId,
1009
+ runId: input.runId,
1010
+ operationId: input.operationId,
1011
+ batchSize: input.batchSize,
1012
+ accountAdminAuthorization: input.accountAdminAuthorization,
1013
+ };
1014
+ const rows = (await scopedDb.execute(sql`
1015
+ SELECT run_document_default_collection_backfill(
1016
+ ${JSON.stringify(command)}::jsonb
1017
+ ) AS result
1018
+ `)) as unknown as Array<{ result: unknown }>;
1019
+ const row = rows[0];
1020
+ if (!row) throw new Error("Document Default collection backfill returned no result");
1021
+ return DocumentDefaultCollectionBackfill.parse(row.result);
1022
+ },
1023
+ );
1024
+ }
1025
+
1026
+ export async function reclassifyDocumentAuthority(
1027
+ db: Database,
1028
+ input: ReclassifyDocumentAuthorityInput,
1029
+ ) {
1030
+ return await withDocumentAccountRls(
1031
+ db,
1032
+ input.accountId,
1033
+ input.workspaceId,
1034
+ { viewerSubjectId: input.actorSubjectId },
1035
+ async (scopedDb) => {
1036
+ const command = {
1037
+ accountId: input.accountId,
1038
+ workspaceId: input.workspaceId,
1039
+ documentId: input.documentId,
1040
+ operationId: input.operationId,
1041
+ actorSubjectId: input.actorSubjectId,
1042
+ expectedAuthority: input.expectedAuthority,
1043
+ targetAuthorityKind: input.targetAuthorityKind,
1044
+ accountAdminAuthorization: input.accountAdminAuthorization,
1045
+ };
1046
+ const rows = (await scopedDb.execute(sql`
1047
+ SELECT reclassify_document_authority(${JSON.stringify(command)}::jsonb) AS result
1048
+ `)) as unknown as Array<{ result: unknown }>;
1049
+ const row = rows[0];
1050
+ if (!row) throw new Error("Document authority reclassification returned no result");
1051
+ return DocumentAuthorityReclassification.parse(row.result);
1052
+ },
1053
+ );
1054
+ }
1055
+
1056
+ export async function listDocumentAuthorityReclassifications(
1057
+ db: Database,
1058
+ input: {
1059
+ accountId: string;
1060
+ workspaceId: string;
1061
+ documentId: string;
1062
+ actorSubjectId: string;
1063
+ limit?: number | undefined;
1064
+ cursor?: string | undefined;
1065
+ },
1066
+ ) {
1067
+ const limit = input.limit ?? 50;
1068
+ if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
1069
+ throw new Error("document authority receipt limit must be between 1 and 100");
1070
+ }
1071
+ const cursor = input.cursor
1072
+ ? decodeDocumentAuthorityReclassificationCursor(input.cursor, input)
1073
+ : null;
1074
+ return await withDocumentAccountRls(
1075
+ db,
1076
+ input.accountId,
1077
+ input.workspaceId,
1078
+ { viewerSubjectId: input.actorSubjectId },
1079
+ async (scopedDb) => {
1080
+ const rows = (await scopedDb.execute(sql`
1081
+ SELECT list_document_authority_reclassifications(
1082
+ ${input.accountId}::uuid,
1083
+ ${input.workspaceId}::uuid,
1084
+ ${input.actorSubjectId},
1085
+ ${input.documentId}::uuid,
1086
+ ${limit + 1}::integer,
1087
+ ${cursor?.createdAt ?? null}::timestamptz,
1088
+ ${cursor?.operationId ?? null}::uuid
1089
+ ) AS result
1090
+ `)) as unknown as Array<{ result: unknown }>;
1091
+ const parsed = rows.map((row) => DocumentAuthorityReclassification.parse(row.result));
1092
+ const hasMore = parsed.length > limit;
1093
+ const receipts = parsed.slice(0, limit);
1094
+ const tail = hasMore ? receipts.at(-1) : null;
1095
+ return ListDocumentAuthorityReclassificationsResponse.parse({
1096
+ receipts,
1097
+ hasMore,
1098
+ nextCursor: tail
1099
+ ? encodeDocumentAuthorityReclassificationCursor(input, {
1100
+ createdAt: tail.createdAt,
1101
+ operationId: tail.operationId,
1102
+ })
1103
+ : null,
1104
+ });
1105
+ },
1106
+ );
1107
+ }
1108
+
1109
+ type DocumentAuthorityReclassificationCursor = {
1110
+ createdAt: string;
1111
+ operationId: string;
1112
+ };
1113
+
1114
+ function documentAuthorityReclassificationCursorScope(input: {
1115
+ accountId: string;
1116
+ workspaceId: string;
1117
+ documentId: string;
1118
+ actorSubjectId: string;
1119
+ }): string {
1120
+ return createHash("sha256")
1121
+ .update(
1122
+ JSON.stringify([
1123
+ "document_authority_reclassification_cursor",
1124
+ 1,
1125
+ input.accountId,
1126
+ input.workspaceId,
1127
+ input.documentId,
1128
+ input.actorSubjectId,
1129
+ ]),
1130
+ "utf8",
1131
+ )
1132
+ .digest("hex")
1133
+ .slice(0, 32);
1134
+ }
1135
+
1136
+ function encodeDocumentAuthorityReclassificationCursor(
1137
+ scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
1138
+ cursor: DocumentAuthorityReclassificationCursor,
1139
+ ): string {
1140
+ return Buffer.from(
1141
+ JSON.stringify({
1142
+ v: 1,
1143
+ s: documentAuthorityReclassificationCursorScope(scope),
1144
+ t: cursor.createdAt,
1145
+ i: cursor.operationId,
1146
+ }),
1147
+ "utf8",
1148
+ ).toString("base64url");
1149
+ }
1150
+
1151
+ function decodeDocumentAuthorityReclassificationCursor(
1152
+ value: string,
1153
+ scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
1154
+ ): DocumentAuthorityReclassificationCursor {
1155
+ try {
1156
+ if (!value || value.length > 1_024) throw new Error("cursor length");
1157
+ const bytes = Buffer.from(value, "base64url");
1158
+ if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
1159
+ const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
1160
+ if (
1161
+ Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
1162
+ parsed.v !== 1 ||
1163
+ parsed.s !== documentAuthorityReclassificationCursorScope(scope) ||
1164
+ typeof parsed.t !== "string" ||
1165
+ !Number.isFinite(new Date(parsed.t).getTime()) ||
1166
+ typeof parsed.i !== "string" ||
1167
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(parsed.i)
1168
+ ) {
1169
+ throw new Error("cursor payload");
1170
+ }
1171
+ return { createdAt: parsed.t, operationId: parsed.i };
1172
+ } catch (error) {
1173
+ throw new Error("invalid document authority receipt cursor", { cause: error });
1174
+ }
1175
+ }
1176
+
932
1177
  export async function addDocumentToBase(
933
1178
  db: Database,
934
1179
  input: AddDocumentRequest & {
@@ -967,7 +1212,18 @@ export async function addDocumentToBase(
967
1212
  if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
968
1213
  const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
969
1214
  if (!base) throw new Error(`Document base not found: ${input.baseId}`);
970
- const file = await requireReadyFile(scopedDb, input.workspaceId, input.fileId);
1215
+ const initiatingSubjectId = cleanString(input.initiatingSubjectId ?? null);
1216
+ const createdBy = cleanString(input.createdBy ?? null);
1217
+ if (initiatingSubjectId && createdBy && initiatingSubjectId !== createdBy) {
1218
+ throw new Error("document file authority must match the exact initiating subject");
1219
+ }
1220
+ const fileAuthoritySubjectId = initiatingSubjectId ?? createdBy ?? null;
1221
+ const file = await requireReadyFile(scopedDb, {
1222
+ accountId: input.accountId,
1223
+ workspaceId: input.workspaceId,
1224
+ subjectId: fileAuthoritySubjectId,
1225
+ fileId: input.fileId,
1226
+ });
971
1227
  const knowledgeSourceIdentity = cleanString(input.knowledgeSourceIdentity ?? null);
972
1228
  if (knowledgeSourceIdentity && knowledgeSourceIdentity.length > 512) {
973
1229
  throw new Error("knowledge source document identity exceeds 512 characters");
@@ -1029,36 +1285,53 @@ export async function addDocumentToBase(
1029
1285
  .returning();
1030
1286
  return mapDocument(updated ?? existing);
1031
1287
  }
1032
- const [row] = await scopedDb
1033
- .insert(schema.documents)
1034
- .values({
1035
- accountId: input.accountId,
1036
- workspaceId: input.workspaceId,
1037
- baseId: input.baseId,
1038
- fileId: input.fileId,
1039
- status: "queued",
1040
- title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
1041
- parser: DEFAULT_DOCUMENT_PARSER,
1042
- sourceKind: input.sourceKind ?? "manual_upload",
1043
- sourceUri: cleanString(input.sourceUri) ?? null,
1044
- sourceExternalId: cleanString(input.sourceExternalId) ?? null,
1045
- sourceTitle: cleanString(input.sourceTitle) ?? null,
1046
- sourceAuthor: cleanString(input.sourceAuthor) ?? null,
1047
- sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
1048
- sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
1049
- sourceVersion: cleanString(input.sourceVersion) ?? null,
1050
- knowledgeSourceIdentity,
1051
- aclTags: cleanStringArray(input.aclTags),
1052
- authorityKind: authority.kind,
1053
- authorityWorkspaceId: authority.workspaceId,
1054
- authoritySubjectId: authority.subjectId,
1055
- visibility: authority.kind === "personal" ? "private" : "workspace",
1056
- agentAccess: input.agentAccess ?? true,
1057
- createdBy: input.createdBy ?? null,
1058
- curationStatus: input.curationStatus ?? "none",
1059
- updatedAt: now,
1060
- })
1061
- .returning();
1288
+ const documentId = randomUUID();
1289
+ const row = await scopedDb.transaction(async (tx) => {
1290
+ const userAuthority =
1291
+ authority.kind === "personal"
1292
+ ? await createPersonalDocumentAuthority(tx, {
1293
+ accountId: input.accountId,
1294
+ workspaceId: input.workspaceId,
1295
+ subjectId: authority.subjectId!,
1296
+ documentId,
1297
+ })
1298
+ : null;
1299
+ const [inserted] = await tx
1300
+ .insert(schema.documents)
1301
+ .values({
1302
+ id: documentId,
1303
+ accountId: input.accountId,
1304
+ workspaceId: input.workspaceId,
1305
+ baseId: input.baseId,
1306
+ fileId: input.fileId,
1307
+ status: "queued",
1308
+ title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
1309
+ parser: DEFAULT_DOCUMENT_PARSER,
1310
+ sourceKind: input.sourceKind ?? "manual_upload",
1311
+ sourceUri: cleanString(input.sourceUri) ?? null,
1312
+ sourceExternalId: cleanString(input.sourceExternalId) ?? null,
1313
+ sourceTitle: cleanString(input.sourceTitle) ?? null,
1314
+ sourceAuthor: cleanString(input.sourceAuthor) ?? null,
1315
+ sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
1316
+ sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
1317
+ sourceVersion: cleanString(input.sourceVersion) ?? null,
1318
+ knowledgeSourceIdentity,
1319
+ aclTags: cleanStringArray(input.aclTags),
1320
+ authorityKind: authority.kind,
1321
+ authorityWorkspaceId: userAuthority ? null : authority.workspaceId,
1322
+ authoritySubjectId: authority.subjectId,
1323
+ authorityId: userAuthority?.authorityId ?? null,
1324
+ ownerOrganizationMembershipId: userAuthority?.ownerOrganizationMembershipId ?? null,
1325
+ originWorkspaceId: input.workspaceId,
1326
+ visibility: authority.kind === "personal" ? "private" : "workspace",
1327
+ agentAccess: input.agentAccess ?? true,
1328
+ createdBy: fileAuthoritySubjectId,
1329
+ curationStatus: input.curationStatus ?? "none",
1330
+ updatedAt: now,
1331
+ })
1332
+ .returning();
1333
+ return inserted;
1334
+ });
1062
1335
  if (!row) throw new Error("Failed to create document");
1063
1336
  return mapDocument(row);
1064
1337
  },
@@ -1092,7 +1365,7 @@ export async function moveDocumentToBase(
1092
1365
  .from(schema.documents)
1093
1366
  .where(
1094
1367
  and(
1095
- eq(schema.documents.workspaceId, input.workspaceId),
1368
+ eq(schema.documents.accountId, input.accountId),
1096
1369
  eq(schema.documents.id, input.documentId),
1097
1370
  ...documentAccessConditions(input.workspaceId, input.access),
1098
1371
  ),
@@ -1107,14 +1380,18 @@ export async function moveDocumentToBase(
1107
1380
  throw new Error("document has no suggested base; pass targetBaseId");
1108
1381
  }
1109
1382
  if (targetBaseId === row.baseId) return mapDocument(row);
1110
- const base = await getDocumentBase(scopedDb, input.workspaceId, targetBaseId);
1383
+ // A portable personal Document retains its immutable ingestion workspace
1384
+ // as provenance and physical storage. Management may be authorized from
1385
+ // another same-organization workspace, but filing still targets a base
1386
+ // in that origin workspace rather than silently copying authority/data.
1387
+ const base = await getDocumentBase(scopedDb, row.workspaceId, targetBaseId);
1111
1388
  if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
1112
1389
  const [conflict] = await scopedDb
1113
1390
  .select({ id: schema.documents.id })
1114
1391
  .from(schema.documents)
1115
1392
  .where(
1116
1393
  and(
1117
- eq(schema.documents.workspaceId, input.workspaceId),
1394
+ eq(schema.documents.workspaceId, row.workspaceId),
1118
1395
  eq(schema.documents.baseId, targetBaseId),
1119
1396
  ...(row.knowledgeSourceIdentity
1120
1397
  ? [eq(schema.documents.knowledgeSourceIdentity, row.knowledgeSourceIdentity)]
@@ -1138,7 +1415,7 @@ export async function moveDocumentToBase(
1138
1415
  })
1139
1416
  .where(
1140
1417
  and(
1141
- eq(schema.documents.workspaceId, input.workspaceId),
1418
+ eq(schema.documents.workspaceId, row.workspaceId),
1142
1419
  eq(schema.documents.id, input.documentId),
1143
1420
  ...documentAccessConditions(input.workspaceId, input.access),
1144
1421
  ),
@@ -1150,7 +1427,7 @@ export async function moveDocumentToBase(
1150
1427
  .set({ baseId: targetBaseId })
1151
1428
  .where(
1152
1429
  and(
1153
- eq(schema.documentChunks.workspaceId, input.workspaceId),
1430
+ eq(schema.documentChunks.workspaceId, row.workspaceId),
1154
1431
  eq(schema.documentChunks.documentId, input.documentId),
1155
1432
  ),
1156
1433
  );
@@ -1185,7 +1462,7 @@ export async function deleteDocumentFromBase(
1185
1462
  .from(schema.documents)
1186
1463
  .where(
1187
1464
  and(
1188
- eq(schema.documents.workspaceId, input.workspaceId),
1465
+ eq(schema.documents.accountId, input.accountId),
1189
1466
  eq(schema.documents.id, input.documentId),
1190
1467
  ...documentAccessConditions(input.workspaceId, input.access),
1191
1468
  ),
@@ -1205,7 +1482,7 @@ export async function deleteDocumentFromBase(
1205
1482
  .delete(schema.documents)
1206
1483
  .where(
1207
1484
  and(
1208
- eq(schema.documents.workspaceId, input.workspaceId),
1485
+ eq(schema.documents.accountId, input.accountId),
1209
1486
  eq(schema.documents.id, input.documentId),
1210
1487
  ...documentAccessConditions(input.workspaceId, input.access),
1211
1488
  ),
@@ -1236,6 +1513,28 @@ export async function listDocuments(
1236
1513
  });
1237
1514
  }
1238
1515
 
1516
+ /**
1517
+ * List Documents the human can discover and manage from the requested
1518
+ * workspace. Organization Documents are account-wide, workspace Documents are
1519
+ * local, activated organization-user Documents are portable across the owner's
1520
+ * same-organization workspaces, and legacy personal rows remain anchored to
1521
+ * their ingestion workspace through documentAccessConditions.
1522
+ */
1523
+ export async function listAccessibleDocuments(
1524
+ db: Database,
1525
+ workspaceId: string,
1526
+ access: DocumentAccessFilter,
1527
+ ): Promise<Document[]> {
1528
+ return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
1529
+ const rows = await scopedDb
1530
+ .select()
1531
+ .from(schema.documents)
1532
+ .where(and(...documentAccessConditions(workspaceId, access)))
1533
+ .orderBy(desc(schema.documents.updatedAt), asc(schema.documents.createdAt));
1534
+ return rows.map(mapDocument);
1535
+ });
1536
+ }
1537
+
1239
1538
  /**
1240
1539
  * List newly ready documents in the same effective scope used by agent
1241
1540
  * retrieval. The opaque checkpoint is bound to the account, requesting
@@ -1258,10 +1557,13 @@ export async function listEffectiveIndexedDocuments(
1258
1557
  initiatingSubjectId,
1259
1558
  })
1260
1559
  : 0n;
1261
- const access: DocumentAccessFilter = {
1262
- agentOnly: true,
1263
- viewerSubjectId: initiatingSubjectId,
1264
- };
1560
+ const access = await resolveEffectiveDocumentAccess(db, {
1561
+ accountId: input.accountId,
1562
+ workspaceId: input.workspaceId,
1563
+ initiatingSubjectId,
1564
+ surface: "agent",
1565
+ agentAuthority: input.agentAuthority,
1566
+ });
1265
1567
  const rows = await withDocumentAccountRls(
1266
1568
  db,
1267
1569
  input.accountId,
@@ -1392,11 +1694,55 @@ export async function getDocument(
1392
1694
  .select()
1393
1695
  .from(schema.documents)
1394
1696
  .where(
1395
- and(
1396
- eq(schema.documents.workspaceId, workspaceId),
1397
- eq(schema.documents.id, documentId),
1398
- ...documentAccessConditions(workspaceId, access),
1399
- ),
1697
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1698
+ )
1699
+ .limit(1);
1700
+ return row ? mapDocument(row) : null;
1701
+ });
1702
+ }
1703
+
1704
+ /**
1705
+ * Resolve the immutable source file through Document authority in the requested
1706
+ * workspace. The file remains physically owned by its ingestion workspace;
1707
+ * callers never gain generic access to that workspace's file inventory.
1708
+ */
1709
+ export async function getDocumentOriginalFile(
1710
+ db: Database,
1711
+ input: {
1712
+ accountId: string;
1713
+ workspaceId: string;
1714
+ documentId: string;
1715
+ access: DocumentAccessFilter;
1716
+ },
1717
+ ): Promise<FileAsset | null> {
1718
+ const subjectId = cleanString(input.access.viewerSubjectId ?? null);
1719
+ if (!subjectId || input.access.agentOnly) return null;
1720
+ return await resolveDocumentOriginalFileForSubject(db, {
1721
+ accountId: input.accountId,
1722
+ workspaceId: input.workspaceId,
1723
+ subjectId,
1724
+ documentId: input.documentId,
1725
+ });
1726
+ }
1727
+
1728
+ /**
1729
+ * Internal ingestion-only read used after the worker has independently resolved
1730
+ * and fenced the immutable document authority tuple. It deliberately does not
1731
+ * apply provider retrieval authorization because a new Drive document must be
1732
+ * indexed before its first ACL evidence can be attached. User, API, MCP, and
1733
+ * agent reads must use getDocument/effective retrieval instead.
1734
+ */
1735
+ export async function getDocumentForIndexing(
1736
+ db: Database,
1737
+ workspaceId: string,
1738
+ documentId: string,
1739
+ ): Promise<Document | null> {
1740
+ return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
1741
+ const [row] = await scopedDb
1742
+ .select()
1743
+ .from(schema.documents)
1744
+ .where(
1745
+ and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
1400
1746
  )
1401
1747
  .limit(1);
1402
1748
  return row ? mapDocument(row) : null;
@@ -1412,14 +1758,10 @@ export async function queueDocumentForReindex(
1412
1758
  ): Promise<Document> {
1413
1759
  return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
1414
1760
  const [document] = await scopedDb
1415
- .select({ authorityKind: schema.documents.authorityKind })
1761
+ .select()
1416
1762
  .from(schema.documents)
1417
1763
  .where(
1418
- and(
1419
- eq(schema.documents.workspaceId, workspaceId),
1420
- eq(schema.documents.id, documentId),
1421
- ...documentAccessConditions(workspaceId, access),
1422
- ),
1764
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1423
1765
  )
1424
1766
  .limit(1);
1425
1767
  if (!document) throw new Error(`Document not found: ${documentId}`);
@@ -1432,11 +1774,7 @@ export async function queueDocumentForReindex(
1432
1774
  updatedAt: new Date(),
1433
1775
  })
1434
1776
  .where(
1435
- and(
1436
- eq(schema.documents.workspaceId, workspaceId),
1437
- eq(schema.documents.id, documentId),
1438
- ...documentAccessConditions(workspaceId, access),
1439
- ),
1777
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1440
1778
  )
1441
1779
  .returning();
1442
1780
  if (!row) throw new Error(`Document not found: ${documentId}`);
@@ -1477,7 +1815,12 @@ export async function indexDocumentNow(
1477
1815
  );
1478
1816
  if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
1479
1817
  let document: DocumentRow = loadedDocument;
1480
- const file = await requireReadyFile(db, workspaceId, document.fileId);
1818
+ const file = await requireReadyFile(db, {
1819
+ accountId: document.accountId,
1820
+ workspaceId,
1821
+ subjectId: cleanString(document.createdBy) ?? null,
1822
+ fileId: document.fileId,
1823
+ });
1481
1824
  await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
1482
1825
  await scopedDb
1483
1826
  .update(schema.documents)
@@ -1492,7 +1835,11 @@ export async function indexDocumentNow(
1492
1835
  );
1493
1836
  });
1494
1837
  try {
1495
- const bytes = await objectStorage.getFileBytes(file);
1838
+ const object = await retryWhileMissing(async () =>
1839
+ objectStorage.getObjectBytes(file.objectKey),
1840
+ );
1841
+ if (!object) throw new Error("document source object is missing");
1842
+ const bytes = object.bytes;
1496
1843
  const parsed = await services.parser.parse(bytes, file);
1497
1844
  // Knowledge drops (curationStatus 'pending') are curated between parse and
1498
1845
  // chunking when a provider is enabled, so chunk metadata and base placement
@@ -1604,9 +1951,7 @@ export async function indexDocumentNow(
1604
1951
  // Internal indexing must be able to return a private document to the caller
1605
1952
  // that created/queued it. Public reads remain fail-closed when no subject is
1606
1953
  // supplied; the creator subject is the document's frozen access principal.
1607
- const updated = await getDocument(db, workspaceId, documentId, {
1608
- viewerSubjectId: document.authoritySubjectId,
1609
- });
1954
+ const updated = await getDocumentForIndexing(db, workspaceId, documentId);
1610
1955
  if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
1611
1956
  return updated;
1612
1957
  }
@@ -1750,6 +2095,20 @@ export async function searchDocuments(
1750
2095
  input: DocumentSearchInput,
1751
2096
  services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
1752
2097
  ): Promise<DocumentSearchResult[]> {
2098
+ return (await searchDocumentCandidates(db, input, services)).results;
2099
+ }
2100
+
2101
+ export type DocumentCandidateRelevanceFloor = {
2102
+ vectorScore: number;
2103
+ keywordScore: number;
2104
+ };
2105
+
2106
+ async function searchDocumentCandidates(
2107
+ db: Database,
2108
+ input: DocumentSearchInput,
2109
+ services: Pick<DocumentServices, "embedder">,
2110
+ relevanceFloor?: DocumentCandidateRelevanceFloor,
2111
+ ): Promise<{ results: DocumentSearchResult[]; belowRelevanceFloor: number }> {
1753
2112
  await assertDocumentAccountWorkspace(db, input.accountId, input.workspaceId);
1754
2113
  const mode = input.mode ?? "hybrid";
1755
2114
  const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
@@ -1774,7 +2133,28 @@ export async function searchDocuments(
1774
2133
  if (mode === "keyword" || mode === "hybrid") {
1775
2134
  rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
1776
2135
  }
1777
- return mergeDocumentSearchRows(rows, mode).slice(0, limit);
2136
+ const merged = mergeDocumentSearchRows(rows, mode);
2137
+ return selectDocumentSearchCandidateWindow(merged, limit, relevanceFloor);
2138
+ }
2139
+
2140
+ export function selectDocumentSearchCandidateWindow(
2141
+ merged: DocumentSearchResult[],
2142
+ limit: number,
2143
+ relevanceFloor?: DocumentCandidateRelevanceFloor,
2144
+ ): { results: DocumentSearchResult[]; belowRelevanceFloor: number } {
2145
+ const boundedLimit = Math.min(Math.max(Math.trunc(limit), 1), 50);
2146
+ if (!relevanceFloor) {
2147
+ return { results: merged.slice(0, boundedLimit), belowRelevanceFloor: 0 };
2148
+ }
2149
+ let belowRelevanceFloor = 0;
2150
+ const relevant = merged.filter((result) => {
2151
+ const included =
2152
+ (result.vectorScore !== null && result.vectorScore >= relevanceFloor.vectorScore) ||
2153
+ (result.keywordScore !== null && result.keywordScore >= relevanceFloor.keywordScore);
2154
+ if (!included) belowRelevanceFloor += 1;
2155
+ return included;
2156
+ });
2157
+ return { results: relevant.slice(0, boundedLimit), belowRelevanceFloor };
1778
2158
  }
1779
2159
 
1780
2160
  /**
@@ -1791,6 +2171,13 @@ export async function searchEffectiveDocuments(
1791
2171
  if (!initiatingSubjectId) {
1792
2172
  throw new Error("effective document retrieval requires an initiating subject");
1793
2173
  }
2174
+ const access = await resolveEffectiveDocumentAccess(db, {
2175
+ accountId: input.accountId,
2176
+ workspaceId: input.workspaceId,
2177
+ initiatingSubjectId,
2178
+ surface: input.surface,
2179
+ agentAuthority: input.agentAuthority,
2180
+ });
1794
2181
  return await searchDocuments(
1795
2182
  db,
1796
2183
  {
@@ -1804,10 +2191,7 @@ export async function searchEffectiveDocuments(
1804
2191
  aclTags: input.aclTags,
1805
2192
  // Construct the lower-level access filter here instead of spreading the
1806
2193
  // caller input, so an untyped/legacy access override is always ignored.
1807
- access: {
1808
- viewerSubjectId: initiatingSubjectId,
1809
- ...(input.surface === "agent" ? { agentOnly: true } : {}),
1810
- },
2194
+ access,
1811
2195
  },
1812
2196
  services,
1813
2197
  );
@@ -1825,13 +2209,45 @@ export async function searchEffectiveKnowledge(
1825
2209
  services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
1826
2210
  ): Promise<KnowledgeSearchResponse> {
1827
2211
  const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1828
- const ranked = await searchEffectiveDocuments(
2212
+ const requestedLimit = Math.min(Math.max(input.limit ?? 5, 1), KNOWLEDGE_SEARCH_MAX_RESULTS);
2213
+ const access = await resolveEffectiveDocumentAccess(db, {
2214
+ accountId: input.accountId,
2215
+ workspaceId: input.workspaceId,
2216
+ initiatingSubjectId,
2217
+ surface: input.surface,
2218
+ agentAuthority: input.agentAuthority,
2219
+ });
2220
+ // Pull a bounded surplus so the permission-safe result set can still satisfy
2221
+ // the caller after relevance filtering and exact-content deduplication.
2222
+ const candidateLimit = Math.min(requestedLimit * 4, KNOWLEDGE_SEARCH_MAX_RESULTS);
2223
+ const rankedSelection = await searchDocumentCandidates(
1829
2224
  db,
1830
- { ...input, initiatingSubjectId, surface: "agent" },
2225
+ {
2226
+ accountId: input.accountId,
2227
+ workspaceId: input.workspaceId,
2228
+ query: input.query,
2229
+ limit: candidateLimit,
2230
+ ...(input.baseIds ? { baseIds: input.baseIds } : {}),
2231
+ ...(input.mode ? { mode: input.mode } : {}),
2232
+ ...(input.sourceKinds ? { sourceKinds: input.sourceKinds } : {}),
2233
+ ...(input.aclTags ? { aclTags: input.aclTags } : {}),
2234
+ access,
2235
+ },
1831
2236
  services,
2237
+ {
2238
+ vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
2239
+ keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
2240
+ },
1832
2241
  );
1833
- if (ranked.length === 0) return { results: [] };
1834
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
2242
+ const ranked = rankedSelection.results;
2243
+ if (ranked.length === 0) {
2244
+ return selectKnowledgeSearchResults({
2245
+ candidates: [],
2246
+ rankedCandidateCount: 0,
2247
+ requestedLimit,
2248
+ alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
2249
+ });
2250
+ }
1835
2251
  const current = await withDocumentAccountRls(
1836
2252
  db,
1837
2253
  input.accountId,
@@ -1839,7 +2255,13 @@ export async function searchEffectiveKnowledge(
1839
2255
  access,
1840
2256
  async (scopedDb) =>
1841
2257
  await scopedDb
1842
- .select({ chunk: schema.documentChunks, document: schema.documents })
2258
+ .select({
2259
+ chunk: schema.documentChunks,
2260
+ document: schema.documents,
2261
+ citation: googleDriveCitationProjection(input.workspaceId, access),
2262
+ previousChunkId: knowledgePreviousChunkIdProjection(),
2263
+ nextChunkId: knowledgeNextChunkIdProjection(),
2264
+ })
1843
2265
  .from(schema.documentChunks)
1844
2266
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1845
2267
  .where(
@@ -1856,23 +2278,399 @@ export async function searchEffectiveKnowledge(
1856
2278
  ),
1857
2279
  );
1858
2280
  const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
1859
- return {
1860
- results: ranked.flatMap((rankedResult) => {
2281
+ return selectKnowledgeSearchResults({
2282
+ rankedCandidateCount: ranked.length,
2283
+ requestedLimit,
2284
+ alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
2285
+ candidates: ranked.flatMap((rankedResult) => {
1861
2286
  const row = currentByChunkId.get(rankedResult.chunkId);
1862
2287
  if (!row) return [];
1863
2288
  return [
1864
2289
  {
1865
- record: knowledgeChunkRecord(row.document, row.chunk),
1866
- retrieval: {
1867
- score: rankedResult.score,
1868
- matchType: rankedResult.matchType,
1869
- vectorScore: rankedResult.vectorScore,
1870
- keywordScore: rankedResult.keywordScore,
1871
- },
2290
+ record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
2291
+ previousChunkId: row.previousChunkId,
2292
+ nextChunkId: row.nextChunkId,
2293
+ }),
2294
+ semanticScore: rankedResult.score,
2295
+ matchType: rankedResult.matchType,
2296
+ vectorScore: rankedResult.vectorScore,
2297
+ keywordScore: rankedResult.keywordScore,
1872
2298
  },
1873
2299
  ];
1874
2300
  }),
2301
+ });
2302
+ }
2303
+
2304
+ type KnowledgeSearchCandidate = {
2305
+ record: KnowledgeRecord;
2306
+ semanticScore: number;
2307
+ matchType: DocumentSearchMode;
2308
+ vectorScore: number | null;
2309
+ keywordScore: number | null;
2310
+ };
2311
+
2312
+ type KnowledgeSearchSelectionInput = {
2313
+ candidates: KnowledgeSearchCandidate[];
2314
+ rankedCandidateCount: number;
2315
+ requestedLimit: number;
2316
+ alreadyBelowRelevanceFloor?: number | undefined;
2317
+ now?: Date | undefined;
2318
+ };
2319
+
2320
+ /**
2321
+ * Deterministic, content-safe final selection over already-authorized and
2322
+ * freshly rechecked Knowledge candidates. Exported for boundary tests; callers
2323
+ * must never use it as a substitute for the database authorization pass above.
2324
+ */
2325
+ export function selectKnowledgeSearchResults(
2326
+ input: KnowledgeSearchSelectionInput,
2327
+ ): KnowledgeSearchResponse {
2328
+ const requestedLimit = Math.min(
2329
+ Math.max(Math.trunc(input.requestedLimit), 1),
2330
+ KNOWLEDGE_SEARCH_MAX_RESULTS,
2331
+ );
2332
+ const nowMs = (input.now ?? new Date()).getTime();
2333
+ let belowRelevanceFloor = Math.min(
2334
+ Math.max(0, Math.trunc(input.alreadyBelowRelevanceFloor ?? 0)),
2335
+ KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
2336
+ );
2337
+ const relevant: KnowledgeSearchResult[] = [];
2338
+ for (const candidate of input.candidates) {
2339
+ const relevanceSignals: Array<"vector" | "keyword"> = [];
2340
+ if (
2341
+ candidate.vectorScore !== null &&
2342
+ candidate.vectorScore >= KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE
2343
+ ) {
2344
+ relevanceSignals.push("vector");
2345
+ }
2346
+ if (
2347
+ candidate.keywordScore !== null &&
2348
+ candidate.keywordScore >= KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE
2349
+ ) {
2350
+ relevanceSignals.push("keyword");
2351
+ }
2352
+ if (relevanceSignals.length === 0) {
2353
+ belowRelevanceFloor = Math.min(belowRelevanceFloor + 1, KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS);
2354
+ continue;
2355
+ }
2356
+ const freshness = knowledgeFreshness(candidate.record.quality.freshnessAt, nowMs);
2357
+ const qualityAdjustment = freshness === "current" ? 0.02 : freshness === "aging" ? 0.01 : 0;
2358
+ relevant.push({
2359
+ record: candidate.record,
2360
+ retrieval: {
2361
+ score: roundScore(Math.min(1, candidate.semanticScore + qualityAdjustment)),
2362
+ semanticScore: roundScore(candidate.semanticScore),
2363
+ matchType: candidate.matchType,
2364
+ vectorScore: candidate.vectorScore === null ? null : roundScore(candidate.vectorScore),
2365
+ keywordScore: candidate.keywordScore === null ? null : roundScore(candidate.keywordScore),
2366
+ relevanceSignals,
2367
+ freshness,
2368
+ qualityAdjustment,
2369
+ duplicateCount: 0,
2370
+ },
2371
+ });
2372
+ }
2373
+ relevant.sort(compareKnowledgeSearchResults);
2374
+
2375
+ const deduped: KnowledgeSearchResult[] = [];
2376
+ const byContent = new Map<string, number>();
2377
+ let asDuplicate = 0;
2378
+ for (const result of relevant) {
2379
+ const key = knowledgeTextualContentKey(result.record);
2380
+ const retainedIndex = byContent.get(key);
2381
+ if (retainedIndex === undefined) {
2382
+ byContent.set(key, deduped.length);
2383
+ deduped.push(result);
2384
+ continue;
2385
+ }
2386
+ asDuplicate += 1;
2387
+ const retained = deduped[retainedIndex]!;
2388
+ retained.retrieval.duplicateCount += 1;
2389
+ }
2390
+
2391
+ const forLimit = Math.max(0, deduped.length - requestedLimit);
2392
+ const bounded = deduped.slice(0, requestedLimit);
2393
+ let forResponseBudget = 0;
2394
+ let response = knowledgeSearchResponse({
2395
+ results: bounded,
2396
+ rankedCandidateCount: input.rankedCandidateCount,
2397
+ recheckedCandidateCount: input.candidates.length,
2398
+ belowRelevanceFloor,
2399
+ asDuplicate,
2400
+ forLimit,
2401
+ forResponseBudget,
2402
+ });
2403
+ while (knowledgeResponseBytes(response) > KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES) {
2404
+ if (bounded.length === 0) {
2405
+ throw new Error("knowledge search selection facts exceed the response budget");
2406
+ }
2407
+ bounded.pop();
2408
+ forResponseBudget += 1;
2409
+ response = knowledgeSearchResponse({
2410
+ results: bounded,
2411
+ rankedCandidateCount: input.rankedCandidateCount,
2412
+ recheckedCandidateCount: input.candidates.length,
2413
+ belowRelevanceFloor,
2414
+ asDuplicate,
2415
+ forLimit,
2416
+ forResponseBudget,
2417
+ });
2418
+ }
2419
+ return response;
2420
+ }
2421
+
2422
+ function compareKnowledgeSearchResults(
2423
+ left: KnowledgeSearchResult,
2424
+ right: KnowledgeSearchResult,
2425
+ ): number {
2426
+ return (
2427
+ right.retrieval.score - left.retrieval.score ||
2428
+ right.retrieval.semanticScore - left.retrieval.semanticScore ||
2429
+ (right.retrieval.vectorScore ?? 0) - (left.retrieval.vectorScore ?? 0) ||
2430
+ (right.retrieval.keywordScore ?? 0) - (left.retrieval.keywordScore ?? 0) ||
2431
+ (left.record.id === right.record.id ? 0 : left.record.id < right.record.id ? -1 : 1)
2432
+ );
2433
+ }
2434
+
2435
+ function knowledgeFreshness(value: string, nowMs: number): "current" | "aging" | "stale" {
2436
+ const freshnessMs = Date.parse(value);
2437
+ if (!Number.isFinite(freshnessMs)) return "stale";
2438
+ const ageDays = Math.max(0, (nowMs - freshnessMs) / 86_400_000);
2439
+ if (ageDays <= 90) return "current";
2440
+ return ageDays <= 365 ? "aging" : "stale";
2441
+ }
2442
+
2443
+ function knowledgeTextualContentKey(record: KnowledgeRecord): string {
2444
+ return createHash("sha256")
2445
+ .update("opengeni:knowledge-search-content:v1\0")
2446
+ .update(record.title)
2447
+ .update("\0")
2448
+ .update(
2449
+ JSON.stringify({
2450
+ body: record.content.body,
2451
+ summary: record.content.summary,
2452
+ topics: record.content.topics,
2453
+ }),
2454
+ )
2455
+ .digest("hex");
2456
+ }
2457
+
2458
+ function knowledgeSearchResponse(input: {
2459
+ results: KnowledgeSearchResult[];
2460
+ rankedCandidateCount: number;
2461
+ recheckedCandidateCount: number;
2462
+ belowRelevanceFloor: number;
2463
+ asDuplicate: number;
2464
+ forLimit: number;
2465
+ forResponseBudget: number;
2466
+ }): KnowledgeSearchResponse {
2467
+ const selection = {
2468
+ relevanceFloor: {
2469
+ policy: "any_signal" as const,
2470
+ vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE as 0.52,
2471
+ keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE as 0.01,
2472
+ },
2473
+ dedupe: { policy: "exact_textual_content" as const },
2474
+ candidates: {
2475
+ ranked: input.rankedCandidateCount,
2476
+ rechecked: input.recheckedCandidateCount,
2477
+ omittedOnRecheck: Math.max(0, input.rankedCandidateCount - input.recheckedCandidateCount),
2478
+ },
2479
+ omitted: {
2480
+ belowRelevanceFloor: input.belowRelevanceFloor,
2481
+ asDuplicate: input.asDuplicate,
2482
+ forLimit: input.forLimit,
2483
+ forResponseBudget: input.forResponseBudget,
2484
+ },
2485
+ budget: {
2486
+ maxResults: KNOWLEDGE_SEARCH_MAX_RESULTS as 50,
2487
+ maxResponseBytes: KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES as 65_536,
2488
+ responseBytes: 0,
2489
+ tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
2490
+ estimatedTokens: 0,
2491
+ maxEstimatedTokens: Math.ceil(
2492
+ KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2493
+ ) as 16_384,
2494
+ },
2495
+ };
2496
+ const response: KnowledgeSearchResponse = {
2497
+ results: [...input.results],
2498
+ selection,
1875
2499
  };
2500
+ // The counters themselves contribute a few bytes. Iterate to a fixed point so
2501
+ // responseBytes describes the actual serialized response, not an approximation.
2502
+ for (let index = 0; index < 8; index += 1) {
2503
+ const responseBytes = knowledgeResponseBytes(response);
2504
+ const estimatedTokens = Math.ceil(
2505
+ responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2506
+ );
2507
+ if (
2508
+ response.selection.budget.responseBytes === responseBytes &&
2509
+ response.selection.budget.estimatedTokens === estimatedTokens
2510
+ ) {
2511
+ break;
2512
+ }
2513
+ response.selection.budget.responseBytes = responseBytes;
2514
+ response.selection.budget.estimatedTokens = estimatedTokens;
2515
+ }
2516
+ return response;
2517
+ }
2518
+
2519
+ function knowledgeResponseBytes(response: KnowledgeSearchResponse): number {
2520
+ return Buffer.byteLength(JSON.stringify(response), "utf8");
2521
+ }
2522
+
2523
+ type KnowledgeBrowseEntry = {
2524
+ record: KnowledgeRecord;
2525
+ cursorAfter: string;
2526
+ };
2527
+
2528
+ /**
2529
+ * Bound a browse page without skipping records. Tail records omitted for the
2530
+ * response budget remain behind the returned cursor. If one complete record is
2531
+ * itself too large, return a deterministic discovery projection; knowledge_get
2532
+ * remains the freshly authorized full-record path.
2533
+ */
2534
+ export function selectKnowledgeBrowseRecords(input: {
2535
+ entries: KnowledgeBrowseEntry[];
2536
+ hasMoreAfterEntries: boolean;
2537
+ }): KnowledgeBrowseResponse {
2538
+ const entries = input.entries.slice(0, KNOWLEDGE_BROWSE_MAX_LIMIT);
2539
+ const selected = entries.map((entry) => entry.record);
2540
+ let omittedForResponseBudget = 0;
2541
+ let compactedRecordCount = 0;
2542
+ let response = knowledgeBrowseResponse({
2543
+ records: selected,
2544
+ nextCursor: input.hasMoreAfterEntries ? (entries.at(-1)?.cursorAfter ?? null) : null,
2545
+ hasMore: input.hasMoreAfterEntries,
2546
+ omittedForResponseBudget,
2547
+ compactedRecordCount,
2548
+ });
2549
+ while (
2550
+ selected.length > 1 &&
2551
+ knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
2552
+ ) {
2553
+ selected.pop();
2554
+ omittedForResponseBudget += 1;
2555
+ response = knowledgeBrowseResponse({
2556
+ records: selected,
2557
+ nextCursor: entries[selected.length - 1]?.cursorAfter ?? null,
2558
+ hasMore: true,
2559
+ omittedForResponseBudget,
2560
+ compactedRecordCount,
2561
+ });
2562
+ }
2563
+ if (
2564
+ selected.length === 1 &&
2565
+ knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
2566
+ ) {
2567
+ selected[0] = compactKnowledgeBrowseRecord(selected[0]!);
2568
+ compactedRecordCount = 1;
2569
+ response = knowledgeBrowseResponse({
2570
+ records: selected,
2571
+ nextCursor:
2572
+ omittedForResponseBudget > 0 || input.hasMoreAfterEntries
2573
+ ? (entries[0]?.cursorAfter ?? null)
2574
+ : null,
2575
+ hasMore: omittedForResponseBudget > 0 || input.hasMoreAfterEntries,
2576
+ omittedForResponseBudget,
2577
+ compactedRecordCount,
2578
+ });
2579
+ }
2580
+ if (knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES) {
2581
+ throw new Error("knowledge browse discovery projection exceeds the response budget");
2582
+ }
2583
+ return response;
2584
+ }
2585
+
2586
+ function compactKnowledgeBrowseRecord(record: KnowledgeRecord): KnowledgeRecord {
2587
+ const fields = new Set<KnowledgeRecord["projection"]["fields"][number]>([
2588
+ ...record.projection.fields,
2589
+ "content.body",
2590
+ "content.summary",
2591
+ "content.topics",
2592
+ "content.metadata",
2593
+ "provenance.source.uri",
2594
+ "provenance.source.externalId",
2595
+ "provenance.source.title",
2596
+ "provenance.source.author",
2597
+ "provenance.source.version",
2598
+ "provenance.citation",
2599
+ ]);
2600
+ return {
2601
+ ...record,
2602
+ content: {
2603
+ format: "markdown",
2604
+ body: null,
2605
+ summary: null,
2606
+ topics: [],
2607
+ metadata: {},
2608
+ },
2609
+ provenance: {
2610
+ ...record.provenance,
2611
+ source: {
2612
+ ...record.provenance.source,
2613
+ uri: null,
2614
+ externalId: null,
2615
+ title: null,
2616
+ author: null,
2617
+ version: null,
2618
+ },
2619
+ citation: null,
2620
+ },
2621
+ links: record.links.filter((link) => link.target.kind === "knowledge"),
2622
+ projection: {
2623
+ truncated: true,
2624
+ fields: [...fields].sort(),
2625
+ },
2626
+ };
2627
+ }
2628
+
2629
+ function knowledgeBrowseResponse(input: {
2630
+ records: KnowledgeRecord[];
2631
+ nextCursor: string | null;
2632
+ hasMore: boolean;
2633
+ omittedForResponseBudget: number;
2634
+ compactedRecordCount: number;
2635
+ }): KnowledgeBrowseResponse {
2636
+ const response: KnowledgeBrowseResponse = {
2637
+ records: [...input.records],
2638
+ nextCursor: input.nextCursor,
2639
+ hasMore: input.hasMore,
2640
+ selection: {
2641
+ omitted: { forResponseBudget: input.omittedForResponseBudget },
2642
+ compactedRecordCount: input.compactedRecordCount,
2643
+ budget: {
2644
+ maxResults: KNOWLEDGE_BROWSE_MAX_LIMIT as 50,
2645
+ maxResponseBytes: KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES as 65_536,
2646
+ responseBytes: 0,
2647
+ tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
2648
+ estimatedTokens: 0,
2649
+ maxEstimatedTokens: Math.ceil(
2650
+ KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2651
+ ) as 16_384,
2652
+ },
2653
+ },
2654
+ };
2655
+ for (let index = 0; index < 8; index += 1) {
2656
+ const responseBytes = knowledgeBrowseResponseBytes(response);
2657
+ const estimatedTokens = Math.ceil(
2658
+ responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2659
+ );
2660
+ if (
2661
+ response.selection.budget.responseBytes === responseBytes &&
2662
+ response.selection.budget.estimatedTokens === estimatedTokens
2663
+ ) {
2664
+ break;
2665
+ }
2666
+ response.selection.budget.responseBytes = responseBytes;
2667
+ response.selection.budget.estimatedTokens = estimatedTokens;
2668
+ }
2669
+ return response;
2670
+ }
2671
+
2672
+ function knowledgeBrowseResponseBytes(response: KnowledgeBrowseResponse): number {
2673
+ return Buffer.byteLength(JSON.stringify(response), "utf8");
1876
2674
  }
1877
2675
 
1878
2676
  /** Fetch one stable Knowledge record with a fresh authorization check. */
@@ -1882,12 +2680,20 @@ export async function getEffectiveKnowledgeRecord(
1882
2680
  accountId: string;
1883
2681
  workspaceId: string;
1884
2682
  initiatingSubjectId: string;
2683
+ surface?: "human" | "agent" | undefined;
1885
2684
  id: string;
2685
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
1886
2686
  },
1887
2687
  ): Promise<KnowledgeRecord | null> {
1888
2688
  const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1889
2689
  const target = parseKnowledgeRecordId(input.id);
1890
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
2690
+ const access = await resolveEffectiveDocumentAccess(db, {
2691
+ accountId: input.accountId,
2692
+ workspaceId: input.workspaceId,
2693
+ initiatingSubjectId,
2694
+ surface: input.surface ?? "agent",
2695
+ agentAuthority: input.agentAuthority,
2696
+ });
1891
2697
  return await withDocumentAccountRls(
1892
2698
  db,
1893
2699
  input.accountId,
@@ -1895,9 +2701,21 @@ export async function getEffectiveKnowledgeRecord(
1895
2701
  access,
1896
2702
  async (scopedDb) => {
1897
2703
  if (target.kind === "document") {
1898
- const [document] = await scopedDb
1899
- .select()
2704
+ const [row] = await scopedDb
2705
+ .select({
2706
+ document: schema.documents,
2707
+ citation: googleDriveCitationProjection(input.workspaceId, access),
2708
+ firstChunkId: schema.documentChunks.id,
2709
+ })
1900
2710
  .from(schema.documents)
2711
+ .leftJoin(
2712
+ schema.documentChunks,
2713
+ and(
2714
+ eq(schema.documentChunks.accountId, schema.documents.accountId),
2715
+ eq(schema.documentChunks.documentId, schema.documents.id),
2716
+ eq(schema.documentChunks.chunkIndex, 0),
2717
+ ),
2718
+ )
1901
2719
  .where(
1902
2720
  and(
1903
2721
  eq(schema.documents.accountId, input.accountId),
@@ -1907,10 +2725,16 @@ export async function getEffectiveKnowledgeRecord(
1907
2725
  ),
1908
2726
  )
1909
2727
  .limit(1);
1910
- return document ? knowledgeDocumentRecord(document) : null;
2728
+ return row ? knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId) : null;
1911
2729
  }
1912
2730
  const [row] = await scopedDb
1913
- .select({ chunk: schema.documentChunks, document: schema.documents })
2731
+ .select({
2732
+ chunk: schema.documentChunks,
2733
+ document: schema.documents,
2734
+ citation: googleDriveCitationProjection(input.workspaceId, access),
2735
+ previousChunkId: knowledgePreviousChunkIdProjection(),
2736
+ nextChunkId: knowledgeNextChunkIdProjection(),
2737
+ })
1914
2738
  .from(schema.documentChunks)
1915
2739
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1916
2740
  .where(
@@ -1923,7 +2747,12 @@ export async function getEffectiveKnowledgeRecord(
1923
2747
  ),
1924
2748
  )
1925
2749
  .limit(1);
1926
- return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
2750
+ return row
2751
+ ? knowledgeChunkRecord(row.document, row.chunk, row.citation, {
2752
+ previousChunkId: row.previousChunkId,
2753
+ nextChunkId: row.nextChunkId,
2754
+ })
2755
+ : null;
1927
2756
  },
1928
2757
  );
1929
2758
  }
@@ -1954,7 +2783,7 @@ export async function browseEffectiveKnowledge(
1954
2783
  if (parent && (topic || sourceKinds.length > 0)) {
1955
2784
  throw new Error("knowledge browse document contents do not accept topic/source filters");
1956
2785
  }
1957
- const cursorScope = {
2786
+ const cursorScope: KnowledgeBrowseCursorScope = {
1958
2787
  accountId: input.accountId,
1959
2788
  workspaceId: input.workspaceId,
1960
2789
  initiatingSubjectId,
@@ -1962,11 +2791,15 @@ export async function browseEffectiveKnowledge(
1962
2791
  topic,
1963
2792
  sourceKinds,
1964
2793
  };
1965
- const after = input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
1966
- if (parent && after > 2_147_483_648n) {
1967
- throw new Error("invalid knowledge browse cursor");
1968
- }
1969
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
2794
+ const topLevelAfter =
2795
+ !parent && input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
2796
+ const access = await resolveEffectiveDocumentAccess(db, {
2797
+ accountId: input.accountId,
2798
+ workspaceId: input.workspaceId,
2799
+ initiatingSubjectId,
2800
+ surface: input.surface ?? "agent",
2801
+ agentAuthority: input.agentAuthority,
2802
+ });
1970
2803
  return await withDocumentAccountRls(
1971
2804
  db,
1972
2805
  input.accountId,
@@ -1975,7 +2808,7 @@ export async function browseEffectiveKnowledge(
1975
2808
  async (scopedDb) => {
1976
2809
  if (parent) {
1977
2810
  const [authorizedParent] = await scopedDb
1978
- .select({ id: schema.documents.id })
2811
+ .select({ id: schema.documents.id, indexSequence: schema.documents.indexSequence })
1979
2812
  .from(schema.documents)
1980
2813
  .where(
1981
2814
  and(
@@ -1986,9 +2819,30 @@ export async function browseEffectiveKnowledge(
1986
2819
  ),
1987
2820
  )
1988
2821
  .limit(1);
1989
- if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
2822
+ if (!authorizedParent) {
2823
+ return selectKnowledgeBrowseRecords({ entries: [], hasMoreAfterEntries: false });
2824
+ }
2825
+ if (authorizedParent.indexSequence === null) {
2826
+ throw new Error("ready knowledge document is missing its index revision");
2827
+ }
2828
+ const parentCursorScope: KnowledgeBrowseCursorScope = {
2829
+ ...cursorScope,
2830
+ parentRevision: authorizedParent.indexSequence.toString(),
2831
+ };
2832
+ const after = input.cursor
2833
+ ? decodeKnowledgeBrowseCursor(input.cursor, parentCursorScope)
2834
+ : 0n;
2835
+ if (after > 2_147_483_648n) {
2836
+ throw new Error("invalid knowledge browse cursor");
2837
+ }
1990
2838
  const rows = await scopedDb
1991
- .select({ chunk: schema.documentChunks, document: schema.documents })
2839
+ .select({
2840
+ chunk: schema.documentChunks,
2841
+ document: schema.documents,
2842
+ citation: googleDriveCitationProjection(input.workspaceId, access),
2843
+ previousChunkId: knowledgePreviousChunkIdProjection(),
2844
+ nextChunkId: knowledgeNextChunkIdProjection(),
2845
+ })
1992
2846
  .from(schema.documentChunks)
1993
2847
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1994
2848
  .where(
@@ -1997,6 +2851,7 @@ export async function browseEffectiveKnowledge(
1997
2851
  eq(schema.documentChunks.documentId, parent.id),
1998
2852
  gt(schema.documentChunks.chunkIndex, Number(after) - 1),
1999
2853
  eq(schema.documents.status, "ready"),
2854
+ eq(schema.documents.indexSequence, authorizedParent.indexSequence),
2000
2855
  ...documentAccessConditions(input.workspaceId, access),
2001
2856
  ),
2002
2857
  )
@@ -2004,44 +2859,63 @@ export async function browseEffectiveKnowledge(
2004
2859
  .limit(limit + 1);
2005
2860
  const hasMore = rows.length > limit;
2006
2861
  const page = rows.slice(0, limit);
2007
- const last = page.at(-1)?.chunk.chunkIndex;
2008
- return {
2009
- records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
2010
- nextCursor:
2011
- hasMore && last !== undefined
2012
- ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last + 1))
2013
- : null,
2014
- hasMore,
2015
- };
2862
+ return selectKnowledgeBrowseRecords({
2863
+ entries: page.map((row) => ({
2864
+ record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
2865
+ previousChunkId: row.previousChunkId,
2866
+ nextChunkId: row.nextChunkId,
2867
+ }),
2868
+ cursorAfter: encodeKnowledgeBrowseCursor(
2869
+ parentCursorScope,
2870
+ BigInt(row.chunk.chunkIndex + 1),
2871
+ ),
2872
+ })),
2873
+ hasMoreAfterEntries: hasMore,
2874
+ });
2016
2875
  }
2017
2876
 
2018
2877
  const conditions: SQL[] = [
2019
2878
  eq(schema.documents.accountId, input.accountId),
2020
2879
  eq(schema.documents.status, "ready"),
2021
2880
  isNotNull(schema.documents.indexSequence),
2022
- gt(schema.documents.indexSequence, after),
2881
+ gt(schema.documents.indexSequence, topLevelAfter),
2023
2882
  ...documentAccessConditions(input.workspaceId, access),
2024
2883
  ];
2025
2884
  if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
2026
2885
  if (sourceKinds.length > 0)
2027
2886
  conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
2028
2887
  const rows = await scopedDb
2029
- .select()
2888
+ .select({
2889
+ document: schema.documents,
2890
+ citation: googleDriveCitationProjection(input.workspaceId, access),
2891
+ firstChunkId: schema.documentChunks.id,
2892
+ })
2030
2893
  .from(schema.documents)
2894
+ .leftJoin(
2895
+ schema.documentChunks,
2896
+ and(
2897
+ eq(schema.documentChunks.accountId, schema.documents.accountId),
2898
+ eq(schema.documentChunks.documentId, schema.documents.id),
2899
+ eq(schema.documentChunks.chunkIndex, 0),
2900
+ ),
2901
+ )
2031
2902
  .where(and(...conditions))
2032
2903
  .orderBy(asc(schema.documents.indexSequence))
2033
2904
  .limit(limit + 1);
2034
2905
  const hasMore = rows.length > limit;
2035
2906
  const page = rows.slice(0, limit);
2036
- const last = page.at(-1)?.indexSequence;
2037
- return {
2038
- records: page.map(knowledgeDocumentRecord),
2039
- nextCursor:
2040
- hasMore && last !== undefined && last !== null
2041
- ? encodeKnowledgeBrowseCursor(cursorScope, last)
2042
- : null,
2043
- hasMore,
2044
- };
2907
+ return selectKnowledgeBrowseRecords({
2908
+ entries: page.map((row) => {
2909
+ if (row.document.indexSequence === null) {
2910
+ throw new Error("ready knowledge document is missing its index revision");
2911
+ }
2912
+ return {
2913
+ record: knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId),
2914
+ cursorAfter: encodeKnowledgeBrowseCursor(cursorScope, row.document.indexSequence),
2915
+ };
2916
+ }),
2917
+ hasMoreAfterEntries: hasMore,
2918
+ });
2045
2919
  },
2046
2920
  );
2047
2921
  }
@@ -2051,6 +2925,7 @@ type KnowledgeBrowseCursorScope = {
2051
2925
  workspaceId: string;
2052
2926
  initiatingSubjectId: string;
2053
2927
  parentId: string | null;
2928
+ parentRevision?: string | null | undefined;
2054
2929
  topic: string | null;
2055
2930
  sourceKinds: readonly string[];
2056
2931
  };
@@ -2060,8 +2935,9 @@ export function encodeKnowledgeBrowseCursor(
2060
2935
  position: bigint,
2061
2936
  ): string {
2062
2937
  if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
2938
+ const version = knowledgeBrowseCursorVersion(scope);
2063
2939
  return Buffer.from(
2064
- JSON.stringify({ v: 1, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
2940
+ JSON.stringify({ v: version, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
2065
2941
  "utf8",
2066
2942
  ).toString("base64url");
2067
2943
  }
@@ -2077,9 +2953,10 @@ export function decodeKnowledgeBrowseCursor(
2077
2953
  const bytes = Buffer.from(value, "base64url");
2078
2954
  if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
2079
2955
  const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
2956
+ const expectedVersion = knowledgeBrowseCursorVersion(scope);
2080
2957
  if (
2081
2958
  Object.keys(parsed).sort().join(",") !== "q,s,v" ||
2082
- parsed.v !== 1 ||
2959
+ parsed.v !== expectedVersion ||
2083
2960
  typeof parsed.s !== "string" ||
2084
2961
  typeof parsed.q !== "string" ||
2085
2962
  !/^(0|[1-9][0-9]*)$/.test(parsed.q)
@@ -2104,8 +2981,9 @@ export function decodeKnowledgeBrowseCursor(
2104
2981
  }
2105
2982
 
2106
2983
  function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
2107
- return createHash("sha256")
2108
- .update("opengeni:knowledge-browse-cursor:v1\0")
2984
+ const version = knowledgeBrowseCursorVersion(scope);
2985
+ const hash = createHash("sha256")
2986
+ .update(`opengeni:knowledge-browse-cursor:v${version}\0`)
2109
2987
  .update(scope.accountId)
2110
2988
  .update("\0")
2111
2989
  .update(scope.workspaceId)
@@ -2113,13 +2991,23 @@ function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
2113
2991
  .update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
2114
2992
  .update("\0")
2115
2993
  .update(scope.parentId ?? "")
2116
- .update("\0")
2994
+ .update("\0");
2995
+ if (version === 2) hash.update(scope.parentRevision!).update("\0");
2996
+ return hash
2117
2997
  .update(scope.topic ?? "")
2118
2998
  .update("\0")
2119
2999
  .update([...scope.sourceKinds].sort().join("\0"))
2120
3000
  .digest("hex");
2121
3001
  }
2122
3002
 
3003
+ function knowledgeBrowseCursorVersion(scope: KnowledgeBrowseCursorScope): 1 | 2 {
3004
+ if (!scope.parentId) return 1;
3005
+ if (!scope.parentRevision || !/^[1-9][0-9]*$/.test(scope.parentRevision)) {
3006
+ throw new Error("knowledge browse parent cursor requires an exact document revision");
3007
+ }
3008
+ return 2;
3009
+ }
3010
+
2123
3011
  function parseKnowledgeRecordId(value: string): {
2124
3012
  kind: "document" | "document_chunk";
2125
3013
  id: string;
@@ -2132,7 +3020,11 @@ function parseKnowledgeRecordId(value: string): {
2132
3020
  return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
2133
3021
  }
2134
3022
 
2135
- function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect): KnowledgeRecord {
3023
+ function knowledgeDocumentRecord(
3024
+ document: typeof schema.documents.$inferSelect,
3025
+ citation: unknown = null,
3026
+ firstChunkId: string | null = null,
3027
+ ): KnowledgeRecord {
2136
3028
  if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2137
3029
  const projected = projectKnowledgeRecord({
2138
3030
  title: document.title,
@@ -2151,10 +3043,24 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
2151
3043
  provenance: {
2152
3044
  source: projected.source,
2153
3045
  indexedAt: document.indexedAt.toISOString(),
3046
+ citation: parseKnowledgeProviderCitation(citation),
2154
3047
  },
2155
3048
  lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2156
3049
  quality: knowledgeQuality(document),
2157
- links: knowledgeSourceLinks(projected.source.uri),
3050
+ links: [
3051
+ ...(firstChunkId
3052
+ ? [
3053
+ {
3054
+ relation: "contents" as const,
3055
+ target: {
3056
+ kind: "knowledge" as const,
3057
+ id: `document_chunk:${firstChunkId}` as const,
3058
+ },
3059
+ },
3060
+ ]
3061
+ : []),
3062
+ ...knowledgeSourceLinks(projected.source.uri),
3063
+ ],
2158
3064
  projection: projected.projection,
2159
3065
  };
2160
3066
  }
@@ -2162,6 +3068,11 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
2162
3068
  function knowledgeChunkRecord(
2163
3069
  document: typeof schema.documents.$inferSelect,
2164
3070
  chunk: typeof schema.documentChunks.$inferSelect,
3071
+ citation: unknown = null,
3072
+ traversal: {
3073
+ previousChunkId: string | null;
3074
+ nextChunkId: string | null;
3075
+ } = { previousChunkId: null, nextChunkId: null },
2165
3076
  ): KnowledgeRecord {
2166
3077
  if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2167
3078
  const projected = projectKnowledgeRecord({
@@ -2181,11 +3092,37 @@ function knowledgeChunkRecord(
2181
3092
  provenance: {
2182
3093
  source: projected.source,
2183
3094
  indexedAt: document.indexedAt.toISOString(),
3095
+ citation: parseKnowledgeProviderCitation(citation),
2184
3096
  },
2185
3097
  lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2186
3098
  quality: knowledgeQuality(document),
2187
3099
  links: [
2188
- { relation: "parent", target: { kind: "knowledge", id: `document:${document.id}` } },
3100
+ {
3101
+ relation: "parent",
3102
+ target: { kind: "knowledge", id: `document:${document.id}` },
3103
+ },
3104
+ ...(traversal.previousChunkId
3105
+ ? [
3106
+ {
3107
+ relation: "previous" as const,
3108
+ target: {
3109
+ kind: "knowledge" as const,
3110
+ id: `document_chunk:${traversal.previousChunkId}` as const,
3111
+ },
3112
+ },
3113
+ ]
3114
+ : []),
3115
+ ...(traversal.nextChunkId
3116
+ ? [
3117
+ {
3118
+ relation: "next" as const,
3119
+ target: {
3120
+ kind: "knowledge" as const,
3121
+ id: `document_chunk:${traversal.nextChunkId}` as const,
3122
+ },
3123
+ },
3124
+ ]
3125
+ : []),
2189
3126
  ...knowledgeSourceLinks(projected.source.uri),
2190
3127
  ],
2191
3128
  projection: projected.projection,
@@ -2221,6 +3158,35 @@ function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"
2221
3158
  return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
2222
3159
  }
2223
3160
 
3161
+ /**
3162
+ * These structural targets are selected inside the same authorization-scoped
3163
+ * transaction as their owning record. Only opaque ids are projected; titles,
3164
+ * source fields, and content require a subsequent freshly authorized get.
3165
+ */
3166
+ function knowledgePreviousChunkIdProjection(): SQL<string | null> {
3167
+ return sql<string | null>`(
3168
+ select knowledge_previous_chunk.id
3169
+ from document_chunks knowledge_previous_chunk
3170
+ where knowledge_previous_chunk.account_id = ${schema.documentChunks.accountId}
3171
+ and knowledge_previous_chunk.document_id = ${schema.documentChunks.documentId}
3172
+ and knowledge_previous_chunk.chunk_index < ${schema.documentChunks.chunkIndex}
3173
+ order by knowledge_previous_chunk.chunk_index desc
3174
+ limit 1
3175
+ )`;
3176
+ }
3177
+
3178
+ function knowledgeNextChunkIdProjection(): SQL<string | null> {
3179
+ return sql<string | null>`(
3180
+ select knowledge_next_chunk.id
3181
+ from document_chunks knowledge_next_chunk
3182
+ where knowledge_next_chunk.account_id = ${schema.documentChunks.accountId}
3183
+ and knowledge_next_chunk.document_id = ${schema.documentChunks.documentId}
3184
+ and knowledge_next_chunk.chunk_index > ${schema.documentChunks.chunkIndex}
3185
+ order by knowledge_next_chunk.chunk_index asc
3186
+ limit 1
3187
+ )`;
3188
+ }
3189
+
2224
3190
  async function vectorSearchDocuments(
2225
3191
  db: Database,
2226
3192
  input: DocumentSearchInput,
@@ -2259,12 +3225,13 @@ async function vectorSearchDocuments(
2259
3225
  authorityKind: schema.documents.authorityKind,
2260
3226
  authorityWorkspaceId: schema.documents.authorityWorkspaceId,
2261
3227
  authoritySubjectId: schema.documents.authoritySubjectId,
3228
+ citation: googleDriveCitationProjection(input.workspaceId, input.access),
2262
3229
  distance,
2263
3230
  })
2264
3231
  .from(schema.documentChunks)
2265
3232
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
2266
3233
  .where(and(...documentSearchConditions(input, services.embedder.model)))
2267
- .orderBy(distance)
3234
+ .orderBy(distance, asc(schema.documentChunks.id))
2268
3235
  .limit(limit),
2269
3236
  );
2270
3237
  return rows.map((row) => ({
@@ -2309,6 +3276,7 @@ async function keywordSearchDocuments(
2309
3276
  authorityKind: schema.documents.authorityKind,
2310
3277
  authorityWorkspaceId: schema.documents.authorityWorkspaceId,
2311
3278
  authoritySubjectId: schema.documents.authoritySubjectId,
3279
+ citation: googleDriveCitationProjection(input.workspaceId, input.access),
2312
3280
  rank,
2313
3281
  })
2314
3282
  .from(schema.documentChunks)
@@ -2319,7 +3287,7 @@ async function keywordSearchDocuments(
2319
3287
  sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
2320
3288
  ),
2321
3289
  )
2322
- .orderBy(desc(rank))
3290
+ .orderBy(desc(rank), asc(schema.documentChunks.id))
2323
3291
  .limit(limit),
2324
3292
  );
2325
3293
  return rows.map((row) => ({
@@ -2365,6 +3333,7 @@ export async function getDocumentChunk(
2365
3333
  authorityKind: schema.documents.authorityKind,
2366
3334
  authorityWorkspaceId: schema.documents.authorityWorkspaceId,
2367
3335
  authoritySubjectId: schema.documents.authoritySubjectId,
3336
+ citation: googleDriveCitationProjection(workspaceId, access),
2368
3337
  })
2369
3338
  .from(schema.documentChunks)
2370
3339
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
@@ -2468,6 +3437,33 @@ async function assertDocumentAccountWorkspace(
2468
3437
  return context;
2469
3438
  }
2470
3439
 
3440
+ export async function resolveEffectiveDocumentAccess(
3441
+ _db: Database,
3442
+ input: {
3443
+ accountId: string;
3444
+ workspaceId: string;
3445
+ initiatingSubjectId: string;
3446
+ surface: "human" | "agent";
3447
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
3448
+ },
3449
+ ): Promise<DocumentAccessFilter> {
3450
+ if (input.surface === "human") {
3451
+ return { viewerSubjectId: input.initiatingSubjectId };
3452
+ }
3453
+ return {
3454
+ agentOnly: true,
3455
+ viewerSubjectId: input.initiatingSubjectId,
3456
+ authorizedPersonalAttempt: input.agentAuthority
3457
+ ? {
3458
+ accountId: input.accountId,
3459
+ workspaceId: input.workspaceId,
3460
+ sessionId: input.agentAuthority.sessionId,
3461
+ attemptId: input.agentAuthority.attemptId,
3462
+ }
3463
+ : undefined,
3464
+ };
3465
+ }
3466
+
2471
3467
  /**
2472
3468
  * Visibility/agent scoping shared by every document read path. Fail-closed:
2473
3469
  * with no filter supplied, private documents are invisible.
@@ -2482,32 +3478,70 @@ function documentAccessConditions(
2482
3478
  eq(schema.documents.authorityWorkspaceId, workspaceId),
2483
3479
  );
2484
3480
  const viewer = cleanString(access?.viewerSubjectId ?? null);
3481
+ const personalAttempt = access?.authorizedPersonalAttempt;
3482
+ const authorizedPersonal = personalAttempt
3483
+ ? sql`${schema.documents.id} IN (
3484
+ SELECT resolve_session_attempt_personal_document_reads(
3485
+ ${personalAttempt.accountId}::uuid,
3486
+ ${personalAttempt.workspaceId}::uuid,
3487
+ ${personalAttempt.sessionId}::uuid,
3488
+ ${personalAttempt.attemptId}::uuid
3489
+ )
3490
+ )`
3491
+ : sql`false`;
2485
3492
  const personal = viewer
2486
3493
  ? and(
2487
3494
  eq(schema.documents.authorityKind, "personal"),
2488
- eq(schema.documents.authorityWorkspaceId, workspaceId),
2489
3495
  eq(schema.documents.authoritySubjectId, viewer),
3496
+ access?.agentOnly
3497
+ ? (or(
3498
+ and(
3499
+ isNull(schema.documents.authorityId),
3500
+ eq(schema.documents.authorityWorkspaceId, workspaceId),
3501
+ ),
3502
+ and(isNotNull(schema.documents.authorityId), authorizedPersonal),
3503
+ ) ?? authorizedPersonal)
3504
+ : (or(
3505
+ eq(schema.documents.authorityWorkspaceId, workspaceId),
3506
+ isNull(schema.documents.authorityWorkspaceId),
3507
+ ) ?? eq(schema.documents.authorityWorkspaceId, workspaceId)),
2490
3508
  )
2491
3509
  : undefined;
2492
3510
  const authority = viewer
2493
3511
  ? (or(organization, workspace, personal) ?? organization)
2494
3512
  : (or(organization, workspace) ?? organization);
3513
+ const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
3514
+ const providerAuthorization = sql`google_drive_file_authorized(
3515
+ ${schema.documents.accountId},
3516
+ ${workspaceId}::uuid,
3517
+ ${viewerSql},
3518
+ ${schema.documents.fileId}
3519
+ )`;
2495
3520
  if (access?.agentOnly) {
2496
- return [eq(schema.documents.agentAccess, true), authority];
3521
+ return [eq(schema.documents.agentAccess, true), authority, providerAuthorization];
2497
3522
  }
2498
- return [authority];
3523
+ return [authority, providerAuthorization];
2499
3524
  }
2500
3525
 
2501
3526
  function documentMatchesAccess(
2502
3527
  document: Pick<
2503
3528
  DocumentAccessRecord,
2504
- "authorityKind" | "authorityWorkspaceId" | "authoritySubjectId" | "agentAccess"
3529
+ | "id"
3530
+ | "authorityId"
3531
+ | "authorityKind"
3532
+ | "authorityWorkspaceId"
3533
+ | "authoritySubjectId"
3534
+ | "agentAccess"
2505
3535
  >,
2506
3536
  workspaceId: string,
2507
3537
  access: DocumentAccessFilter | undefined,
2508
3538
  ): boolean {
2509
3539
  if (access?.agentOnly) {
2510
- return document.agentAccess && canViewDocument(document, access.viewerSubjectId, workspaceId);
3540
+ return (
3541
+ document.agentAccess &&
3542
+ (document.authorityKind !== "personal" || document.authorityId === null) &&
3543
+ canViewDocument(document, access.viewerSubjectId, workspaceId)
3544
+ );
2511
3545
  }
2512
3546
  return canViewDocument(document, access?.viewerSubjectId, workspaceId);
2513
3547
  }
@@ -2525,9 +3559,9 @@ function canonicalDocumentAuthoritySubject(value: unknown): string | undefined {
2525
3559
  /**
2526
3560
  * Whether a single already-fetched document is readable in this workspace.
2527
3561
  *
2528
- * Keep this compatibility predicate as strict as SQL/RLS: personal authority
2529
- * remains anchored to its originating workspace, and unknown or incomplete
2530
- * authority tuples deny instead of falling through as workspace-visible.
3562
+ * Keep this compatibility predicate as strict as SQL/RLS: legacy personal
3563
+ * authority remains origin-workspace anchored while activated personal
3564
+ * authority has a null workspace and follows the exact owner within the org.
2531
3565
  */
2532
3566
  export function canViewDocument(
2533
3567
  document: Pick<DocumentAccessRecord, "authorityKind" | "authoritySubjectId"> & {
@@ -2540,21 +3574,31 @@ export function canViewDocument(
2540
3574
  return document.authorityWorkspaceId === null && document.authoritySubjectId === null;
2541
3575
  }
2542
3576
  const normalizedWorkspaceId = cleanString(workspaceId ?? null);
2543
- if (!normalizedWorkspaceId || document.authorityWorkspaceId !== normalizedWorkspaceId) {
3577
+ if (!normalizedWorkspaceId) {
2544
3578
  return false;
2545
3579
  }
2546
3580
  if (document.authorityKind === "workspace") {
2547
- return document.authoritySubjectId === null;
3581
+ return (
3582
+ document.authorityWorkspaceId === normalizedWorkspaceId &&
3583
+ document.authoritySubjectId === null
3584
+ );
2548
3585
  }
2549
3586
  if (document.authorityKind === "personal") {
2550
3587
  const authoritySubjectId = canonicalDocumentAuthoritySubject(document.authoritySubjectId);
2551
3588
  const viewer = canonicalDocumentAuthoritySubject(viewerSubjectId);
2552
- return !!authoritySubjectId && authoritySubjectId === viewer;
3589
+ return (
3590
+ !!authoritySubjectId &&
3591
+ authoritySubjectId === viewer &&
3592
+ (document.authorityWorkspaceId === null ||
3593
+ document.authorityWorkspaceId === normalizedWorkspaceId)
3594
+ );
2553
3595
  }
2554
3596
  return false;
2555
3597
  }
2556
3598
 
2557
3599
  type DocumentAccessRecord = {
3600
+ id: string;
3601
+ authorityId: string | null;
2558
3602
  authorityKind: string;
2559
3603
  authorityWorkspaceId: string | null;
2560
3604
  authoritySubjectId: string | null;
@@ -2606,6 +3650,7 @@ function mapSearchRowBase(row: {
2606
3650
  authorityKind: string;
2607
3651
  authorityWorkspaceId: string | null;
2608
3652
  authoritySubjectId: string | null;
3653
+ citation?: unknown;
2609
3654
  }): SearchRowBase {
2610
3655
  return {
2611
3656
  chunkId: row.chunkId,
@@ -2629,9 +3674,29 @@ function mapSearchRowBase(row: {
2629
3674
  authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
2630
3675
  authorityWorkspaceId: row.authorityWorkspaceId,
2631
3676
  authoritySubjectId: row.authoritySubjectId,
3677
+ citation: parseKnowledgeProviderCitation(row.citation),
2632
3678
  };
2633
3679
  }
2634
3680
 
3681
+ function googleDriveCitationProjection(
3682
+ workspaceId: string,
3683
+ access: DocumentAccessFilter | undefined,
3684
+ ): SQL<unknown> {
3685
+ const viewer = cleanString(access?.viewerSubjectId ?? null);
3686
+ const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
3687
+ return sql`google_drive_document_citation(
3688
+ ${schema.documents.accountId},
3689
+ ${workspaceId}::uuid,
3690
+ ${viewerSql},
3691
+ ${schema.documents.id},
3692
+ ${schema.documents.fileId}
3693
+ )`;
3694
+ }
3695
+
3696
+ function parseKnowledgeProviderCitation(value: unknown): KnowledgeProviderCitationValue | null {
3697
+ return value === null || value === undefined ? null : KnowledgeProviderCitation.parse(value);
3698
+ }
3699
+
2635
3700
  function mergeDocumentSearchRows(
2636
3701
  rows: CombinedSearchRow[],
2637
3702
  mode: DocumentSearchMode,
@@ -2670,7 +3735,8 @@ function mergeDocumentSearchRows(
2670
3735
  right.score - left.score ||
2671
3736
  (right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
2672
3737
  (right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
2673
- left.chunkIndex - right.chunkIndex,
3738
+ left.chunkIndex - right.chunkIndex ||
3739
+ (left.chunkId === right.chunkId ? 0 : left.chunkId < right.chunkId ? -1 : 1),
2674
3740
  );
2675
3741
  }
2676
3742
 
@@ -2763,12 +3829,22 @@ export function deterministicEmbedding(
2763
3829
 
2764
3830
  async function requireReadyFile(
2765
3831
  db: Database,
2766
- workspaceId: string,
2767
- fileId: string,
3832
+ input: {
3833
+ accountId: string;
3834
+ workspaceId: string;
3835
+ subjectId: string | null;
3836
+ fileId: string;
3837
+ },
2768
3838
  ): Promise<FileAsset> {
2769
- const file = await requireFile(db, workspaceId, fileId);
3839
+ const [file] = await getFilesForSubject(db, {
3840
+ accountId: input.accountId,
3841
+ workspaceId: input.workspaceId,
3842
+ subjectId: input.subjectId,
3843
+ fileIds: [input.fileId],
3844
+ });
3845
+ if (!file) throw new Error(`File not found: ${input.fileId}`);
2770
3846
  if (file.status !== "ready") {
2771
- throw new Error(`File ${fileId} is ${file.status}`);
3847
+ throw new Error(`File ${input.fileId} is ${file.status}`);
2772
3848
  }
2773
3849
  return file;
2774
3850
  }
@@ -2916,6 +3992,7 @@ function mapDocument(row: typeof schema.documents.$inferSelect): Document {
2916
3992
  authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
2917
3993
  authorityWorkspaceId: row.authorityWorkspaceId,
2918
3994
  authoritySubjectId: row.authoritySubjectId,
3995
+ authorityId: row.authorityId,
2919
3996
  visibility: normalizeDocumentVisibility(row.visibility),
2920
3997
  createdBy: row.createdBy,
2921
3998
  agentAccess: row.agentAccess,