@opengeni/documents 0.5.41 → 0.7.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -6,6 +6,8 @@ import {
6
6
  CreateDocumentBaseRequest,
7
7
  Document,
8
8
  DocumentAuthorityKind,
9
+ DocumentAuthorityReclassification,
10
+ ListDocumentAuthorityReclassificationsResponse,
9
11
  DocumentBase,
10
12
  DocumentCuration,
11
13
  DocumentCurationStatus,
@@ -17,12 +19,25 @@ import {
17
19
  IndexedDocumentSummary,
18
20
  KnowledgeBrowseResponse,
19
21
  KnowledgeRecord,
22
+ KnowledgeSearchResult,
20
23
  KnowledgeSearchResponse,
21
24
  KnowledgeSourceKind,
22
25
  ListIndexedDocumentsResponse,
26
+ type ReclassifyDocumentAuthorityRequest,
27
+ type RunDocumentDefaultCollectionBackfillRequest,
28
+ DocumentDefaultCollectionBackfill,
29
+ DocumentDefaultCollectionBackfillAudit,
30
+ DocumentDefaultCollectionBackfillOperationAudit,
31
+ DocumentDefaultCollectionBackfillReceiptAudit,
32
+ DocumentDefaultCollectionBackfillRunAudit,
33
+ ListDocumentDefaultCollectionBackfillRunsResponse,
34
+ ListOrganizationDocumentAuthorityReclassificationsResponse,
35
+ OrganizationDocumentAuthorityReclassification,
23
36
  } from "@opengeni/contracts";
24
37
  import {
38
+ createPersonalDocumentAuthority,
25
39
  getFilesForSubject,
40
+ resolveDocumentOriginalFileForSubject,
26
41
  rlsContextForWorkspace,
27
42
  setSubjectRlsContext,
28
43
  withRlsContext,
@@ -31,11 +46,21 @@ import {
31
46
  type Database,
32
47
  } from "@opengeni/db";
33
48
  import * as schema from "@opengeni/db/schema";
34
- import type { ObjectStorage } from "@opengeni/storage";
35
- import { createHash } from "node:crypto";
36
- import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
49
+ import { retryWhileMissing, type ObjectStorage } from "@opengeni/storage";
50
+ import { createHash, randomUUID } from "node:crypto";
51
+ import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from "drizzle-orm";
37
52
  import type OpenAI from "openai";
38
- import { KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS } from "@opengeni/contracts";
53
+ import {
54
+ KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS,
55
+ KNOWLEDGE_BROWSE_MAX_LIMIT,
56
+ KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES,
57
+ KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES,
58
+ KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
59
+ KNOWLEDGE_SEARCH_MAX_RESULTS,
60
+ KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
61
+ KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
62
+ KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
63
+ } from "@opengeni/contracts";
39
64
  import { projectKnowledgeRecord } from "./knowledge-projection";
40
65
 
41
66
  export { projectKnowledgeRecord } from "./knowledge-projection";
@@ -138,6 +163,15 @@ export type DocumentAccessFilter = {
138
163
  * only when the agent carries the creating subject as its viewer subject.
139
164
  */
140
165
  agentOnly?: boolean | undefined;
166
+ /** Exact attempt whose grant snapshot must be revalidated in the content query. */
167
+ authorizedPersonalAttempt?:
168
+ | {
169
+ accountId: string;
170
+ workspaceId: string;
171
+ sessionId: string;
172
+ attemptId: string;
173
+ }
174
+ | undefined;
141
175
  };
142
176
 
143
177
  export type DocumentAuthority = {
@@ -146,6 +180,44 @@ export type DocumentAuthority = {
146
180
  subjectId: string | null;
147
181
  };
148
182
 
183
+ /** JSON-safe projection of the opaque capability minted by the API access layer. */
184
+ export type DocumentAccountAdminAuthorization = Readonly<{
185
+ authorizationId: string;
186
+ accountId: string;
187
+ actorSubjectId: string;
188
+ permission: "account:admin";
189
+ }>;
190
+
191
+ export type ReclassifyDocumentAuthorityInput = ReclassifyDocumentAuthorityRequest & {
192
+ accountId: string;
193
+ workspaceId: string;
194
+ documentId: string;
195
+ actorSubjectId: string;
196
+ accountAdminAuthorization: DocumentAccountAdminAuthorization | null;
197
+ };
198
+
199
+ export type RunDocumentDefaultCollectionBackfillInput =
200
+ RunDocumentDefaultCollectionBackfillRequest & {
201
+ accountId: string;
202
+ workspaceId: string;
203
+ actorSubjectId: string;
204
+ accountAdminAuthorization: DocumentAccountAdminAuthorization;
205
+ };
206
+
207
+ export type DocumentMigrationAuditInput = {
208
+ accountId: string;
209
+ workspaceId: string;
210
+ actorSubjectId: string;
211
+ accountAdminAuthorization: DocumentAccountAdminAuthorization;
212
+ limit?: number | undefined;
213
+ cursor?: string | undefined;
214
+ };
215
+
216
+ export type AgentDocumentAuthorityContext = {
217
+ sessionId: string;
218
+ attemptId: string;
219
+ };
220
+
149
221
  export type DocumentInventoryStatusCounts = Record<DocumentStatus, number>;
150
222
  export type DocumentInventorySourceKindCounts = Record<KnowledgeSourceKind, number>;
151
223
  export type DocumentInventoryAuthorityKindCounts = Record<DocumentAuthorityKind, number>;
@@ -196,6 +268,7 @@ export type EffectiveDocumentSearchInput = Omit<DocumentSearchInput, "access"> &
196
268
  initiatingSubjectId: string;
197
269
  /** Agent retrieval additionally enforces documents.agent_access. */
198
270
  surface: "human" | "agent";
271
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
199
272
  };
200
273
 
201
274
  export type ListEffectiveIndexedDocumentsInput = {
@@ -205,6 +278,7 @@ export type ListEffectiveIndexedDocumentsInput = {
205
278
  initiatingSubjectId: string;
206
279
  checkpoint?: string | undefined;
207
280
  limit?: number | undefined;
281
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
208
282
  };
209
283
 
210
284
  export type EffectiveKnowledgeBrowseInput = {
@@ -212,12 +286,15 @@ export type EffectiveKnowledgeBrowseInput = {
212
286
  workspaceId: string;
213
287
  /** Immutable human subject accepted for the logical request/turn. */
214
288
  initiatingSubjectId: string;
289
+ /** Human inspection bypasses agent_access but still uses exact subject/RLS authority. */
290
+ surface?: "human" | "agent" | undefined;
215
291
  /** Omit to browse top-level documents; pass a document record id for chunks. */
216
292
  parentId?: string | undefined;
217
293
  topic?: string | undefined;
218
294
  sourceKinds?: KnowledgeSourceKind[] | undefined;
219
295
  cursor?: string | undefined;
220
296
  limit?: number | undefined;
297
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
221
298
  };
222
299
 
223
300
  export type DocumentIndexHooks = {
@@ -931,6 +1008,438 @@ export async function listDocumentBasesEnsuringDefault(
931
1008
  return await listDocumentBases(db, input.workspaceId);
932
1009
  }
933
1010
 
1011
+ export async function runDocumentDefaultCollectionBackfill(
1012
+ db: Database,
1013
+ input: RunDocumentDefaultCollectionBackfillInput,
1014
+ ) {
1015
+ return await withDocumentAccountRls(
1016
+ db,
1017
+ input.accountId,
1018
+ input.workspaceId,
1019
+ { viewerSubjectId: input.actorSubjectId },
1020
+ async (scopedDb) => {
1021
+ const command = {
1022
+ accountId: input.accountId,
1023
+ workspaceId: input.workspaceId,
1024
+ actorSubjectId: input.actorSubjectId,
1025
+ runId: input.runId,
1026
+ operationId: input.operationId,
1027
+ batchSize: input.batchSize,
1028
+ accountAdminAuthorization: input.accountAdminAuthorization,
1029
+ };
1030
+ const rows = (await scopedDb.execute(sql`
1031
+ SELECT run_document_default_collection_backfill(
1032
+ ${JSON.stringify(command)}::jsonb
1033
+ ) AS result
1034
+ `)) as unknown as Array<{ result: unknown }>;
1035
+ const row = rows[0];
1036
+ if (!row) throw new Error("Document Default collection backfill returned no result");
1037
+ return DocumentDefaultCollectionBackfill.parse(row.result);
1038
+ },
1039
+ );
1040
+ }
1041
+
1042
+ export async function listDocumentDefaultCollectionBackfillRuns(
1043
+ db: Database,
1044
+ input: DocumentMigrationAuditInput,
1045
+ ) {
1046
+ const limit = validateDocumentMigrationAuditLimit(input.limit);
1047
+ const cursor = input.cursor
1048
+ ? decodeDocumentMigrationAuditCursor(input.cursor, ["backfill-runs", input.accountId], true)
1049
+ : null;
1050
+ return await withDocumentAccountRls(
1051
+ db,
1052
+ input.accountId,
1053
+ input.workspaceId,
1054
+ { viewerSubjectId: input.actorSubjectId },
1055
+ async (scopedDb) => {
1056
+ const command = {
1057
+ accountId: input.accountId,
1058
+ workspaceId: input.workspaceId,
1059
+ actorSubjectId: input.actorSubjectId,
1060
+ accountAdminAuthorization: input.accountAdminAuthorization,
1061
+ limit: limit + 1,
1062
+ beforeStartedAt: cursor?.timestamp ?? null,
1063
+ beforeRunId: cursor?.id ?? null,
1064
+ };
1065
+ const rows = (await scopedDb.execute(sql`
1066
+ SELECT list_document_default_collection_backfill_runs(
1067
+ ${JSON.stringify(command)}::jsonb
1068
+ ) AS result
1069
+ `)) as unknown as Array<{ result: unknown }>;
1070
+ const parsed = rows.map((row) => DocumentDefaultCollectionBackfillRunAudit.parse(row.result));
1071
+ const hasMore = parsed.length > limit;
1072
+ const runs = parsed.slice(0, limit);
1073
+ const tail = hasMore ? runs.at(-1) : null;
1074
+ return ListDocumentDefaultCollectionBackfillRunsResponse.parse({
1075
+ runs,
1076
+ hasMore,
1077
+ nextCursor: tail
1078
+ ? encodeDocumentMigrationAuditCursor(["backfill-runs", input.accountId], {
1079
+ timestamp: tail.startedAt,
1080
+ id: tail.runId,
1081
+ })
1082
+ : null,
1083
+ });
1084
+ },
1085
+ );
1086
+ }
1087
+
1088
+ export async function getDocumentDefaultCollectionBackfillAudit(
1089
+ db: Database,
1090
+ input: DocumentMigrationAuditInput & {
1091
+ runId: string;
1092
+ operationCursor?: string | undefined;
1093
+ receiptCursor?: string | undefined;
1094
+ },
1095
+ ) {
1096
+ const limit = validateDocumentMigrationAuditLimit(input.limit);
1097
+ const operationCursor = input.operationCursor
1098
+ ? decodeDocumentMigrationAuditCursor(
1099
+ input.operationCursor,
1100
+ ["backfill-operations", input.accountId, input.runId],
1101
+ true,
1102
+ )
1103
+ : null;
1104
+ const receiptCursor = input.receiptCursor
1105
+ ? decodeDocumentMigrationAuditCursor(
1106
+ input.receiptCursor,
1107
+ ["backfill-receipts", input.accountId, input.runId],
1108
+ false,
1109
+ )
1110
+ : null;
1111
+ return await withDocumentAccountRls(
1112
+ db,
1113
+ input.accountId,
1114
+ input.workspaceId,
1115
+ { viewerSubjectId: input.actorSubjectId },
1116
+ async (scopedDb) => {
1117
+ const command = {
1118
+ accountId: input.accountId,
1119
+ workspaceId: input.workspaceId,
1120
+ actorSubjectId: input.actorSubjectId,
1121
+ accountAdminAuthorization: input.accountAdminAuthorization,
1122
+ runId: input.runId,
1123
+ limit,
1124
+ operationBeforeCreatedAt: operationCursor?.timestamp ?? null,
1125
+ operationBeforeId: operationCursor?.id ?? null,
1126
+ receiptAfterWorkspaceId: receiptCursor?.id ?? null,
1127
+ };
1128
+ const rows = (await scopedDb.execute(sql`
1129
+ SELECT get_document_default_collection_backfill_audit(
1130
+ ${JSON.stringify(command)}::jsonb
1131
+ ) AS result
1132
+ `)) as unknown as Array<{ result: unknown }>;
1133
+ const raw = rows[0]?.result;
1134
+ if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
1135
+ throw new Error("document Default collection backfill audit returned no result");
1136
+ }
1137
+ const value = raw as Record<string, unknown>;
1138
+ const run = DocumentDefaultCollectionBackfillRunAudit.parse(value.run);
1139
+ const operations = Array.isArray(value.operations)
1140
+ ? value.operations.map((operation) =>
1141
+ DocumentDefaultCollectionBackfillOperationAudit.parse(operation),
1142
+ )
1143
+ : [];
1144
+ const receipts = Array.isArray(value.receipts)
1145
+ ? value.receipts.map((receipt) =>
1146
+ DocumentDefaultCollectionBackfillReceiptAudit.parse(receipt),
1147
+ )
1148
+ : [];
1149
+ const operationsHasMore = operations.length > limit;
1150
+ const receiptsHasMore = receipts.length > limit;
1151
+ const operationPage = operations.slice(0, limit);
1152
+ const receiptPage = receipts.slice(0, limit);
1153
+ const operationTail = operationsHasMore ? operationPage.at(-1) : null;
1154
+ const receiptTail = receiptsHasMore ? receiptPage.at(-1) : null;
1155
+ return DocumentDefaultCollectionBackfillAudit.parse({
1156
+ run,
1157
+ operations: operationPage,
1158
+ receipts: receiptPage,
1159
+ operationsHasMore,
1160
+ operationsNextCursor: operationTail
1161
+ ? encodeDocumentMigrationAuditCursor(
1162
+ ["backfill-operations", input.accountId, input.runId],
1163
+ { timestamp: operationTail.createdAt, id: operationTail.operationId },
1164
+ )
1165
+ : null,
1166
+ receiptsHasMore,
1167
+ receiptsNextCursor: receiptTail
1168
+ ? encodeDocumentMigrationAuditCursor(
1169
+ ["backfill-receipts", input.accountId, input.runId],
1170
+ { timestamp: null, id: receiptTail.workspaceId },
1171
+ )
1172
+ : null,
1173
+ });
1174
+ },
1175
+ );
1176
+ }
1177
+
1178
+ export async function listOrganizationDocumentAuthorityReclassifications(
1179
+ db: Database,
1180
+ input: DocumentMigrationAuditInput,
1181
+ ) {
1182
+ const limit = validateDocumentMigrationAuditLimit(input.limit);
1183
+ const cursor = input.cursor
1184
+ ? decodeDocumentMigrationAuditCursor(
1185
+ input.cursor,
1186
+ ["organization-reclassifications", input.accountId],
1187
+ true,
1188
+ )
1189
+ : null;
1190
+ return await withDocumentAccountRls(
1191
+ db,
1192
+ input.accountId,
1193
+ input.workspaceId,
1194
+ { viewerSubjectId: input.actorSubjectId },
1195
+ async (scopedDb) => {
1196
+ const command = {
1197
+ accountId: input.accountId,
1198
+ workspaceId: input.workspaceId,
1199
+ actorSubjectId: input.actorSubjectId,
1200
+ accountAdminAuthorization: input.accountAdminAuthorization,
1201
+ limit: limit + 1,
1202
+ beforeCreatedAt: cursor?.timestamp ?? null,
1203
+ beforeOperationId: cursor?.id ?? null,
1204
+ };
1205
+ const rows = (await scopedDb.execute(sql`
1206
+ SELECT list_organization_document_authority_reclassifications(
1207
+ ${JSON.stringify(command)}::jsonb
1208
+ ) AS result
1209
+ `)) as unknown as Array<{ result: unknown }>;
1210
+ const parsed = rows.map((row) =>
1211
+ OrganizationDocumentAuthorityReclassification.parse(row.result),
1212
+ );
1213
+ const hasMore = parsed.length > limit;
1214
+ const receipts = parsed.slice(0, limit);
1215
+ const tail = hasMore ? receipts.at(-1) : null;
1216
+ return ListOrganizationDocumentAuthorityReclassificationsResponse.parse({
1217
+ receipts,
1218
+ hasMore,
1219
+ nextCursor: tail
1220
+ ? encodeDocumentMigrationAuditCursor(
1221
+ ["organization-reclassifications", input.accountId],
1222
+ { timestamp: tail.createdAt, id: tail.operationId },
1223
+ )
1224
+ : null,
1225
+ });
1226
+ },
1227
+ );
1228
+ }
1229
+
1230
+ function validateDocumentMigrationAuditLimit(value: number | undefined): number {
1231
+ const limit = value ?? 50;
1232
+ if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
1233
+ throw new Error("document migration audit limit must be between 1 and 100");
1234
+ }
1235
+ return limit;
1236
+ }
1237
+
1238
+ type DocumentMigrationAuditCursor = { timestamp: string | null; id: string };
1239
+
1240
+ function documentMigrationAuditCursorScope(parts: string[]): string {
1241
+ return createHash("sha256")
1242
+ .update(JSON.stringify(["document-migration-audit", 1, ...parts]), "utf8")
1243
+ .digest("hex")
1244
+ .slice(0, 32);
1245
+ }
1246
+
1247
+ function encodeDocumentMigrationAuditCursor(
1248
+ parts: string[],
1249
+ cursor: DocumentMigrationAuditCursor,
1250
+ ): string {
1251
+ return Buffer.from(
1252
+ JSON.stringify({
1253
+ v: 1,
1254
+ s: documentMigrationAuditCursorScope(parts),
1255
+ t: cursor.timestamp,
1256
+ i: cursor.id,
1257
+ }),
1258
+ "utf8",
1259
+ ).toString("base64url");
1260
+ }
1261
+
1262
+ function decodeDocumentMigrationAuditCursor(
1263
+ value: string,
1264
+ parts: string[],
1265
+ requireTimestamp: boolean,
1266
+ ): DocumentMigrationAuditCursor {
1267
+ try {
1268
+ if (!value || value.length > 1_024) throw new Error("cursor length");
1269
+ const bytes = Buffer.from(value, "base64url");
1270
+ if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
1271
+ const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
1272
+ if (
1273
+ Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
1274
+ parsed.v !== 1 ||
1275
+ parsed.s !== documentMigrationAuditCursorScope(parts) ||
1276
+ typeof parsed.i !== "string" ||
1277
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(
1278
+ parsed.i,
1279
+ ) ||
1280
+ (requireTimestamp
1281
+ ? typeof parsed.t !== "string" || !Number.isFinite(new Date(parsed.t).getTime())
1282
+ : parsed.t !== null)
1283
+ ) {
1284
+ throw new Error("cursor payload");
1285
+ }
1286
+ return { timestamp: typeof parsed.t === "string" ? parsed.t : null, id: parsed.i };
1287
+ } catch (error) {
1288
+ throw new Error("invalid document migration audit cursor", { cause: error });
1289
+ }
1290
+ }
1291
+
1292
+ export async function reclassifyDocumentAuthority(
1293
+ db: Database,
1294
+ input: ReclassifyDocumentAuthorityInput,
1295
+ ) {
1296
+ return await withDocumentAccountRls(
1297
+ db,
1298
+ input.accountId,
1299
+ input.workspaceId,
1300
+ { viewerSubjectId: input.actorSubjectId },
1301
+ async (scopedDb) => {
1302
+ const command = {
1303
+ accountId: input.accountId,
1304
+ workspaceId: input.workspaceId,
1305
+ documentId: input.documentId,
1306
+ operationId: input.operationId,
1307
+ actorSubjectId: input.actorSubjectId,
1308
+ expectedAuthority: input.expectedAuthority,
1309
+ targetAuthorityKind: input.targetAuthorityKind,
1310
+ accountAdminAuthorization: input.accountAdminAuthorization,
1311
+ };
1312
+ const rows = (await scopedDb.execute(sql`
1313
+ SELECT reclassify_document_authority(${JSON.stringify(command)}::jsonb) AS result
1314
+ `)) as unknown as Array<{ result: unknown }>;
1315
+ const row = rows[0];
1316
+ if (!row) throw new Error("Document authority reclassification returned no result");
1317
+ return DocumentAuthorityReclassification.parse(row.result);
1318
+ },
1319
+ );
1320
+ }
1321
+
1322
+ export async function listDocumentAuthorityReclassifications(
1323
+ db: Database,
1324
+ input: {
1325
+ accountId: string;
1326
+ workspaceId: string;
1327
+ documentId: string;
1328
+ actorSubjectId: string;
1329
+ limit?: number | undefined;
1330
+ cursor?: string | undefined;
1331
+ },
1332
+ ) {
1333
+ const limit = input.limit ?? 50;
1334
+ if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
1335
+ throw new Error("document authority receipt limit must be between 1 and 100");
1336
+ }
1337
+ const cursor = input.cursor
1338
+ ? decodeDocumentAuthorityReclassificationCursor(input.cursor, input)
1339
+ : null;
1340
+ return await withDocumentAccountRls(
1341
+ db,
1342
+ input.accountId,
1343
+ input.workspaceId,
1344
+ { viewerSubjectId: input.actorSubjectId },
1345
+ async (scopedDb) => {
1346
+ const rows = (await scopedDb.execute(sql`
1347
+ SELECT list_document_authority_reclassifications(
1348
+ ${input.accountId}::uuid,
1349
+ ${input.workspaceId}::uuid,
1350
+ ${input.actorSubjectId},
1351
+ ${input.documentId}::uuid,
1352
+ ${limit + 1}::integer,
1353
+ ${cursor?.createdAt ?? null}::timestamptz,
1354
+ ${cursor?.operationId ?? null}::uuid
1355
+ ) AS result
1356
+ `)) as unknown as Array<{ result: unknown }>;
1357
+ const parsed = rows.map((row) => DocumentAuthorityReclassification.parse(row.result));
1358
+ const hasMore = parsed.length > limit;
1359
+ const receipts = parsed.slice(0, limit);
1360
+ const tail = hasMore ? receipts.at(-1) : null;
1361
+ return ListDocumentAuthorityReclassificationsResponse.parse({
1362
+ receipts,
1363
+ hasMore,
1364
+ nextCursor: tail
1365
+ ? encodeDocumentAuthorityReclassificationCursor(input, {
1366
+ createdAt: tail.createdAt,
1367
+ operationId: tail.operationId,
1368
+ })
1369
+ : null,
1370
+ });
1371
+ },
1372
+ );
1373
+ }
1374
+
1375
+ type DocumentAuthorityReclassificationCursor = {
1376
+ createdAt: string;
1377
+ operationId: string;
1378
+ };
1379
+
1380
+ function documentAuthorityReclassificationCursorScope(input: {
1381
+ accountId: string;
1382
+ workspaceId: string;
1383
+ documentId: string;
1384
+ actorSubjectId: string;
1385
+ }): string {
1386
+ return createHash("sha256")
1387
+ .update(
1388
+ JSON.stringify([
1389
+ "document_authority_reclassification_cursor",
1390
+ 1,
1391
+ input.accountId,
1392
+ input.workspaceId,
1393
+ input.documentId,
1394
+ input.actorSubjectId,
1395
+ ]),
1396
+ "utf8",
1397
+ )
1398
+ .digest("hex")
1399
+ .slice(0, 32);
1400
+ }
1401
+
1402
+ function encodeDocumentAuthorityReclassificationCursor(
1403
+ scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
1404
+ cursor: DocumentAuthorityReclassificationCursor,
1405
+ ): string {
1406
+ return Buffer.from(
1407
+ JSON.stringify({
1408
+ v: 1,
1409
+ s: documentAuthorityReclassificationCursorScope(scope),
1410
+ t: cursor.createdAt,
1411
+ i: cursor.operationId,
1412
+ }),
1413
+ "utf8",
1414
+ ).toString("base64url");
1415
+ }
1416
+
1417
+ function decodeDocumentAuthorityReclassificationCursor(
1418
+ value: string,
1419
+ scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
1420
+ ): DocumentAuthorityReclassificationCursor {
1421
+ try {
1422
+ if (!value || value.length > 1_024) throw new Error("cursor length");
1423
+ const bytes = Buffer.from(value, "base64url");
1424
+ if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
1425
+ const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
1426
+ if (
1427
+ Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
1428
+ parsed.v !== 1 ||
1429
+ parsed.s !== documentAuthorityReclassificationCursorScope(scope) ||
1430
+ typeof parsed.t !== "string" ||
1431
+ !Number.isFinite(new Date(parsed.t).getTime()) ||
1432
+ typeof parsed.i !== "string" ||
1433
+ !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(parsed.i)
1434
+ ) {
1435
+ throw new Error("cursor payload");
1436
+ }
1437
+ return { createdAt: parsed.t, operationId: parsed.i };
1438
+ } catch (error) {
1439
+ throw new Error("invalid document authority receipt cursor", { cause: error });
1440
+ }
1441
+ }
1442
+
934
1443
  export async function addDocumentToBase(
935
1444
  db: Database,
936
1445
  input: AddDocumentRequest & {
@@ -1042,36 +1551,53 @@ export async function addDocumentToBase(
1042
1551
  .returning();
1043
1552
  return mapDocument(updated ?? existing);
1044
1553
  }
1045
- const [row] = await scopedDb
1046
- .insert(schema.documents)
1047
- .values({
1048
- accountId: input.accountId,
1049
- workspaceId: input.workspaceId,
1050
- baseId: input.baseId,
1051
- fileId: input.fileId,
1052
- status: "queued",
1053
- title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
1054
- parser: DEFAULT_DOCUMENT_PARSER,
1055
- sourceKind: input.sourceKind ?? "manual_upload",
1056
- sourceUri: cleanString(input.sourceUri) ?? null,
1057
- sourceExternalId: cleanString(input.sourceExternalId) ?? null,
1058
- sourceTitle: cleanString(input.sourceTitle) ?? null,
1059
- sourceAuthor: cleanString(input.sourceAuthor) ?? null,
1060
- sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
1061
- sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
1062
- sourceVersion: cleanString(input.sourceVersion) ?? null,
1063
- knowledgeSourceIdentity,
1064
- aclTags: cleanStringArray(input.aclTags),
1065
- authorityKind: authority.kind,
1066
- authorityWorkspaceId: authority.workspaceId,
1067
- authoritySubjectId: authority.subjectId,
1068
- visibility: authority.kind === "personal" ? "private" : "workspace",
1069
- agentAccess: input.agentAccess ?? true,
1070
- createdBy: fileAuthoritySubjectId,
1071
- curationStatus: input.curationStatus ?? "none",
1072
- updatedAt: now,
1073
- })
1074
- .returning();
1554
+ const documentId = randomUUID();
1555
+ const row = await scopedDb.transaction(async (tx) => {
1556
+ const userAuthority =
1557
+ authority.kind === "personal"
1558
+ ? await createPersonalDocumentAuthority(tx, {
1559
+ accountId: input.accountId,
1560
+ workspaceId: input.workspaceId,
1561
+ subjectId: authority.subjectId!,
1562
+ documentId,
1563
+ })
1564
+ : null;
1565
+ const [inserted] = await tx
1566
+ .insert(schema.documents)
1567
+ .values({
1568
+ id: documentId,
1569
+ accountId: input.accountId,
1570
+ workspaceId: input.workspaceId,
1571
+ baseId: input.baseId,
1572
+ fileId: input.fileId,
1573
+ status: "queued",
1574
+ title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
1575
+ parser: DEFAULT_DOCUMENT_PARSER,
1576
+ sourceKind: input.sourceKind ?? "manual_upload",
1577
+ sourceUri: cleanString(input.sourceUri) ?? null,
1578
+ sourceExternalId: cleanString(input.sourceExternalId) ?? null,
1579
+ sourceTitle: cleanString(input.sourceTitle) ?? null,
1580
+ sourceAuthor: cleanString(input.sourceAuthor) ?? null,
1581
+ sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
1582
+ sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
1583
+ sourceVersion: cleanString(input.sourceVersion) ?? null,
1584
+ knowledgeSourceIdentity,
1585
+ aclTags: cleanStringArray(input.aclTags),
1586
+ authorityKind: authority.kind,
1587
+ authorityWorkspaceId: userAuthority ? null : authority.workspaceId,
1588
+ authoritySubjectId: authority.subjectId,
1589
+ authorityId: userAuthority?.authorityId ?? null,
1590
+ ownerOrganizationMembershipId: userAuthority?.ownerOrganizationMembershipId ?? null,
1591
+ originWorkspaceId: input.workspaceId,
1592
+ visibility: authority.kind === "personal" ? "private" : "workspace",
1593
+ agentAccess: input.agentAccess ?? true,
1594
+ createdBy: fileAuthoritySubjectId,
1595
+ curationStatus: input.curationStatus ?? "none",
1596
+ updatedAt: now,
1597
+ })
1598
+ .returning();
1599
+ return inserted;
1600
+ });
1075
1601
  if (!row) throw new Error("Failed to create document");
1076
1602
  return mapDocument(row);
1077
1603
  },
@@ -1105,7 +1631,7 @@ export async function moveDocumentToBase(
1105
1631
  .from(schema.documents)
1106
1632
  .where(
1107
1633
  and(
1108
- eq(schema.documents.workspaceId, input.workspaceId),
1634
+ eq(schema.documents.accountId, input.accountId),
1109
1635
  eq(schema.documents.id, input.documentId),
1110
1636
  ...documentAccessConditions(input.workspaceId, input.access),
1111
1637
  ),
@@ -1120,14 +1646,18 @@ export async function moveDocumentToBase(
1120
1646
  throw new Error("document has no suggested base; pass targetBaseId");
1121
1647
  }
1122
1648
  if (targetBaseId === row.baseId) return mapDocument(row);
1123
- const base = await getDocumentBase(scopedDb, input.workspaceId, targetBaseId);
1649
+ // A portable personal Document retains its immutable ingestion workspace
1650
+ // as provenance and physical storage. Management may be authorized from
1651
+ // another same-organization workspace, but filing still targets a base
1652
+ // in that origin workspace rather than silently copying authority/data.
1653
+ const base = await getDocumentBase(scopedDb, row.workspaceId, targetBaseId);
1124
1654
  if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
1125
1655
  const [conflict] = await scopedDb
1126
1656
  .select({ id: schema.documents.id })
1127
1657
  .from(schema.documents)
1128
1658
  .where(
1129
1659
  and(
1130
- eq(schema.documents.workspaceId, input.workspaceId),
1660
+ eq(schema.documents.workspaceId, row.workspaceId),
1131
1661
  eq(schema.documents.baseId, targetBaseId),
1132
1662
  ...(row.knowledgeSourceIdentity
1133
1663
  ? [eq(schema.documents.knowledgeSourceIdentity, row.knowledgeSourceIdentity)]
@@ -1151,7 +1681,7 @@ export async function moveDocumentToBase(
1151
1681
  })
1152
1682
  .where(
1153
1683
  and(
1154
- eq(schema.documents.workspaceId, input.workspaceId),
1684
+ eq(schema.documents.workspaceId, row.workspaceId),
1155
1685
  eq(schema.documents.id, input.documentId),
1156
1686
  ...documentAccessConditions(input.workspaceId, input.access),
1157
1687
  ),
@@ -1163,7 +1693,7 @@ export async function moveDocumentToBase(
1163
1693
  .set({ baseId: targetBaseId })
1164
1694
  .where(
1165
1695
  and(
1166
- eq(schema.documentChunks.workspaceId, input.workspaceId),
1696
+ eq(schema.documentChunks.workspaceId, row.workspaceId),
1167
1697
  eq(schema.documentChunks.documentId, input.documentId),
1168
1698
  ),
1169
1699
  );
@@ -1198,7 +1728,7 @@ export async function deleteDocumentFromBase(
1198
1728
  .from(schema.documents)
1199
1729
  .where(
1200
1730
  and(
1201
- eq(schema.documents.workspaceId, input.workspaceId),
1731
+ eq(schema.documents.accountId, input.accountId),
1202
1732
  eq(schema.documents.id, input.documentId),
1203
1733
  ...documentAccessConditions(input.workspaceId, input.access),
1204
1734
  ),
@@ -1218,7 +1748,7 @@ export async function deleteDocumentFromBase(
1218
1748
  .delete(schema.documents)
1219
1749
  .where(
1220
1750
  and(
1221
- eq(schema.documents.workspaceId, input.workspaceId),
1751
+ eq(schema.documents.accountId, input.accountId),
1222
1752
  eq(schema.documents.id, input.documentId),
1223
1753
  ...documentAccessConditions(input.workspaceId, input.access),
1224
1754
  ),
@@ -1249,6 +1779,28 @@ export async function listDocuments(
1249
1779
  });
1250
1780
  }
1251
1781
 
1782
+ /**
1783
+ * List Documents the human can discover and manage from the requested
1784
+ * workspace. Organization Documents are account-wide, workspace Documents are
1785
+ * local, activated organization-user Documents are portable across the owner's
1786
+ * same-organization workspaces, and legacy personal rows remain anchored to
1787
+ * their ingestion workspace through documentAccessConditions.
1788
+ */
1789
+ export async function listAccessibleDocuments(
1790
+ db: Database,
1791
+ workspaceId: string,
1792
+ access: DocumentAccessFilter,
1793
+ ): Promise<Document[]> {
1794
+ return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
1795
+ const rows = await scopedDb
1796
+ .select()
1797
+ .from(schema.documents)
1798
+ .where(and(...documentAccessConditions(workspaceId, access)))
1799
+ .orderBy(desc(schema.documents.updatedAt), asc(schema.documents.createdAt));
1800
+ return rows.map(mapDocument);
1801
+ });
1802
+ }
1803
+
1252
1804
  /**
1253
1805
  * List newly ready documents in the same effective scope used by agent
1254
1806
  * retrieval. The opaque checkpoint is bound to the account, requesting
@@ -1271,10 +1823,13 @@ export async function listEffectiveIndexedDocuments(
1271
1823
  initiatingSubjectId,
1272
1824
  })
1273
1825
  : 0n;
1274
- const access: DocumentAccessFilter = {
1275
- agentOnly: true,
1276
- viewerSubjectId: initiatingSubjectId,
1277
- };
1826
+ const access = await resolveEffectiveDocumentAccess(db, {
1827
+ accountId: input.accountId,
1828
+ workspaceId: input.workspaceId,
1829
+ initiatingSubjectId,
1830
+ surface: "agent",
1831
+ agentAuthority: input.agentAuthority,
1832
+ });
1278
1833
  const rows = await withDocumentAccountRls(
1279
1834
  db,
1280
1835
  input.accountId,
@@ -1405,17 +1960,37 @@ export async function getDocument(
1405
1960
  .select()
1406
1961
  .from(schema.documents)
1407
1962
  .where(
1408
- and(
1409
- eq(schema.documents.workspaceId, workspaceId),
1410
- eq(schema.documents.id, documentId),
1411
- ...documentAccessConditions(workspaceId, access),
1412
- ),
1963
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1413
1964
  )
1414
1965
  .limit(1);
1415
1966
  return row ? mapDocument(row) : null;
1416
1967
  });
1417
1968
  }
1418
1969
 
1970
+ /**
1971
+ * Resolve the immutable source file through Document authority in the requested
1972
+ * workspace. The file remains physically owned by its ingestion workspace;
1973
+ * callers never gain generic access to that workspace's file inventory.
1974
+ */
1975
+ export async function getDocumentOriginalFile(
1976
+ db: Database,
1977
+ input: {
1978
+ accountId: string;
1979
+ workspaceId: string;
1980
+ documentId: string;
1981
+ access: DocumentAccessFilter;
1982
+ },
1983
+ ): Promise<FileAsset | null> {
1984
+ const subjectId = cleanString(input.access.viewerSubjectId ?? null);
1985
+ if (!subjectId || input.access.agentOnly) return null;
1986
+ return await resolveDocumentOriginalFileForSubject(db, {
1987
+ accountId: input.accountId,
1988
+ workspaceId: input.workspaceId,
1989
+ subjectId,
1990
+ documentId: input.documentId,
1991
+ });
1992
+ }
1993
+
1419
1994
  /**
1420
1995
  * Internal ingestion-only read used after the worker has independently resolved
1421
1996
  * and fenced the immutable document authority tuple. It deliberately does not
@@ -1449,14 +2024,10 @@ export async function queueDocumentForReindex(
1449
2024
  ): Promise<Document> {
1450
2025
  return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
1451
2026
  const [document] = await scopedDb
1452
- .select({ authorityKind: schema.documents.authorityKind })
2027
+ .select()
1453
2028
  .from(schema.documents)
1454
2029
  .where(
1455
- and(
1456
- eq(schema.documents.workspaceId, workspaceId),
1457
- eq(schema.documents.id, documentId),
1458
- ...documentAccessConditions(workspaceId, access),
1459
- ),
2030
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1460
2031
  )
1461
2032
  .limit(1);
1462
2033
  if (!document) throw new Error(`Document not found: ${documentId}`);
@@ -1469,11 +2040,7 @@ export async function queueDocumentForReindex(
1469
2040
  updatedAt: new Date(),
1470
2041
  })
1471
2042
  .where(
1472
- and(
1473
- eq(schema.documents.workspaceId, workspaceId),
1474
- eq(schema.documents.id, documentId),
1475
- ...documentAccessConditions(workspaceId, access),
1476
- ),
2043
+ and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
1477
2044
  )
1478
2045
  .returning();
1479
2046
  if (!row) throw new Error(`Document not found: ${documentId}`);
@@ -1534,7 +2101,11 @@ export async function indexDocumentNow(
1534
2101
  );
1535
2102
  });
1536
2103
  try {
1537
- const bytes = await objectStorage.getFileBytes(file);
2104
+ const object = await retryWhileMissing(async () =>
2105
+ objectStorage.getObjectBytes(file.objectKey),
2106
+ );
2107
+ if (!object) throw new Error("document source object is missing");
2108
+ const bytes = object.bytes;
1538
2109
  const parsed = await services.parser.parse(bytes, file);
1539
2110
  // Knowledge drops (curationStatus 'pending') are curated between parse and
1540
2111
  // chunking when a provider is enabled, so chunk metadata and base placement
@@ -1790,6 +2361,20 @@ export async function searchDocuments(
1790
2361
  input: DocumentSearchInput,
1791
2362
  services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
1792
2363
  ): Promise<DocumentSearchResult[]> {
2364
+ return (await searchDocumentCandidates(db, input, services)).results;
2365
+ }
2366
+
2367
+ export type DocumentCandidateRelevanceFloor = {
2368
+ vectorScore: number;
2369
+ keywordScore: number;
2370
+ };
2371
+
2372
+ async function searchDocumentCandidates(
2373
+ db: Database,
2374
+ input: DocumentSearchInput,
2375
+ services: Pick<DocumentServices, "embedder">,
2376
+ relevanceFloor?: DocumentCandidateRelevanceFloor,
2377
+ ): Promise<{ results: DocumentSearchResult[]; belowRelevanceFloor: number }> {
1793
2378
  await assertDocumentAccountWorkspace(db, input.accountId, input.workspaceId);
1794
2379
  const mode = input.mode ?? "hybrid";
1795
2380
  const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
@@ -1814,7 +2399,28 @@ export async function searchDocuments(
1814
2399
  if (mode === "keyword" || mode === "hybrid") {
1815
2400
  rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
1816
2401
  }
1817
- return mergeDocumentSearchRows(rows, mode).slice(0, limit);
2402
+ const merged = mergeDocumentSearchRows(rows, mode);
2403
+ return selectDocumentSearchCandidateWindow(merged, limit, relevanceFloor);
2404
+ }
2405
+
2406
+ export function selectDocumentSearchCandidateWindow(
2407
+ merged: DocumentSearchResult[],
2408
+ limit: number,
2409
+ relevanceFloor?: DocumentCandidateRelevanceFloor,
2410
+ ): { results: DocumentSearchResult[]; belowRelevanceFloor: number } {
2411
+ const boundedLimit = Math.min(Math.max(Math.trunc(limit), 1), 50);
2412
+ if (!relevanceFloor) {
2413
+ return { results: merged.slice(0, boundedLimit), belowRelevanceFloor: 0 };
2414
+ }
2415
+ let belowRelevanceFloor = 0;
2416
+ const relevant = merged.filter((result) => {
2417
+ const included =
2418
+ (result.vectorScore !== null && result.vectorScore >= relevanceFloor.vectorScore) ||
2419
+ (result.keywordScore !== null && result.keywordScore >= relevanceFloor.keywordScore);
2420
+ if (!included) belowRelevanceFloor += 1;
2421
+ return included;
2422
+ });
2423
+ return { results: relevant.slice(0, boundedLimit), belowRelevanceFloor };
1818
2424
  }
1819
2425
 
1820
2426
  /**
@@ -1831,6 +2437,13 @@ export async function searchEffectiveDocuments(
1831
2437
  if (!initiatingSubjectId) {
1832
2438
  throw new Error("effective document retrieval requires an initiating subject");
1833
2439
  }
2440
+ const access = await resolveEffectiveDocumentAccess(db, {
2441
+ accountId: input.accountId,
2442
+ workspaceId: input.workspaceId,
2443
+ initiatingSubjectId,
2444
+ surface: input.surface,
2445
+ agentAuthority: input.agentAuthority,
2446
+ });
1834
2447
  return await searchDocuments(
1835
2448
  db,
1836
2449
  {
@@ -1844,10 +2457,7 @@ export async function searchEffectiveDocuments(
1844
2457
  aclTags: input.aclTags,
1845
2458
  // Construct the lower-level access filter here instead of spreading the
1846
2459
  // caller input, so an untyped/legacy access override is always ignored.
1847
- access: {
1848
- viewerSubjectId: initiatingSubjectId,
1849
- ...(input.surface === "agent" ? { agentOnly: true } : {}),
1850
- },
2460
+ access,
1851
2461
  },
1852
2462
  services,
1853
2463
  );
@@ -1865,13 +2475,45 @@ export async function searchEffectiveKnowledge(
1865
2475
  services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
1866
2476
  ): Promise<KnowledgeSearchResponse> {
1867
2477
  const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1868
- const ranked = await searchEffectiveDocuments(
2478
+ const requestedLimit = Math.min(Math.max(input.limit ?? 5, 1), KNOWLEDGE_SEARCH_MAX_RESULTS);
2479
+ const access = await resolveEffectiveDocumentAccess(db, {
2480
+ accountId: input.accountId,
2481
+ workspaceId: input.workspaceId,
2482
+ initiatingSubjectId,
2483
+ surface: input.surface,
2484
+ agentAuthority: input.agentAuthority,
2485
+ });
2486
+ // Pull a bounded surplus so the permission-safe result set can still satisfy
2487
+ // the caller after relevance filtering and exact-content deduplication.
2488
+ const candidateLimit = Math.min(requestedLimit * 4, KNOWLEDGE_SEARCH_MAX_RESULTS);
2489
+ const rankedSelection = await searchDocumentCandidates(
1869
2490
  db,
1870
- { ...input, initiatingSubjectId, surface: "agent" },
2491
+ {
2492
+ accountId: input.accountId,
2493
+ workspaceId: input.workspaceId,
2494
+ query: input.query,
2495
+ limit: candidateLimit,
2496
+ ...(input.baseIds ? { baseIds: input.baseIds } : {}),
2497
+ ...(input.mode ? { mode: input.mode } : {}),
2498
+ ...(input.sourceKinds ? { sourceKinds: input.sourceKinds } : {}),
2499
+ ...(input.aclTags ? { aclTags: input.aclTags } : {}),
2500
+ access,
2501
+ },
1871
2502
  services,
2503
+ {
2504
+ vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
2505
+ keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
2506
+ },
1872
2507
  );
1873
- if (ranked.length === 0) return { results: [] };
1874
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
2508
+ const ranked = rankedSelection.results;
2509
+ if (ranked.length === 0) {
2510
+ return selectKnowledgeSearchResults({
2511
+ candidates: [],
2512
+ rankedCandidateCount: 0,
2513
+ requestedLimit,
2514
+ alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
2515
+ });
2516
+ }
1875
2517
  const current = await withDocumentAccountRls(
1876
2518
  db,
1877
2519
  input.accountId,
@@ -1883,6 +2525,8 @@ export async function searchEffectiveKnowledge(
1883
2525
  chunk: schema.documentChunks,
1884
2526
  document: schema.documents,
1885
2527
  citation: googleDriveCitationProjection(input.workspaceId, access),
2528
+ previousChunkId: knowledgePreviousChunkIdProjection(),
2529
+ nextChunkId: knowledgeNextChunkIdProjection(),
1886
2530
  })
1887
2531
  .from(schema.documentChunks)
1888
2532
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
@@ -1900,23 +2544,399 @@ export async function searchEffectiveKnowledge(
1900
2544
  ),
1901
2545
  );
1902
2546
  const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
1903
- return {
1904
- results: ranked.flatMap((rankedResult) => {
2547
+ return selectKnowledgeSearchResults({
2548
+ rankedCandidateCount: ranked.length,
2549
+ requestedLimit,
2550
+ alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
2551
+ candidates: ranked.flatMap((rankedResult) => {
1905
2552
  const row = currentByChunkId.get(rankedResult.chunkId);
1906
2553
  if (!row) return [];
1907
2554
  return [
1908
2555
  {
1909
- record: knowledgeChunkRecord(row.document, row.chunk, row.citation),
1910
- retrieval: {
1911
- score: rankedResult.score,
1912
- matchType: rankedResult.matchType,
1913
- vectorScore: rankedResult.vectorScore,
1914
- keywordScore: rankedResult.keywordScore,
1915
- },
2556
+ record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
2557
+ previousChunkId: row.previousChunkId,
2558
+ nextChunkId: row.nextChunkId,
2559
+ }),
2560
+ semanticScore: rankedResult.score,
2561
+ matchType: rankedResult.matchType,
2562
+ vectorScore: rankedResult.vectorScore,
2563
+ keywordScore: rankedResult.keywordScore,
1916
2564
  },
1917
2565
  ];
1918
2566
  }),
2567
+ });
2568
+ }
2569
+
2570
+ type KnowledgeSearchCandidate = {
2571
+ record: KnowledgeRecord;
2572
+ semanticScore: number;
2573
+ matchType: DocumentSearchMode;
2574
+ vectorScore: number | null;
2575
+ keywordScore: number | null;
2576
+ };
2577
+
2578
+ type KnowledgeSearchSelectionInput = {
2579
+ candidates: KnowledgeSearchCandidate[];
2580
+ rankedCandidateCount: number;
2581
+ requestedLimit: number;
2582
+ alreadyBelowRelevanceFloor?: number | undefined;
2583
+ now?: Date | undefined;
2584
+ };
2585
+
2586
+ /**
2587
+ * Deterministic, content-safe final selection over already-authorized and
2588
+ * freshly rechecked Knowledge candidates. Exported for boundary tests; callers
2589
+ * must never use it as a substitute for the database authorization pass above.
2590
+ */
2591
+ export function selectKnowledgeSearchResults(
2592
+ input: KnowledgeSearchSelectionInput,
2593
+ ): KnowledgeSearchResponse {
2594
+ const requestedLimit = Math.min(
2595
+ Math.max(Math.trunc(input.requestedLimit), 1),
2596
+ KNOWLEDGE_SEARCH_MAX_RESULTS,
2597
+ );
2598
+ const nowMs = (input.now ?? new Date()).getTime();
2599
+ let belowRelevanceFloor = Math.min(
2600
+ Math.max(0, Math.trunc(input.alreadyBelowRelevanceFloor ?? 0)),
2601
+ KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
2602
+ );
2603
+ const relevant: KnowledgeSearchResult[] = [];
2604
+ for (const candidate of input.candidates) {
2605
+ const relevanceSignals: Array<"vector" | "keyword"> = [];
2606
+ if (
2607
+ candidate.vectorScore !== null &&
2608
+ candidate.vectorScore >= KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE
2609
+ ) {
2610
+ relevanceSignals.push("vector");
2611
+ }
2612
+ if (
2613
+ candidate.keywordScore !== null &&
2614
+ candidate.keywordScore >= KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE
2615
+ ) {
2616
+ relevanceSignals.push("keyword");
2617
+ }
2618
+ if (relevanceSignals.length === 0) {
2619
+ belowRelevanceFloor = Math.min(belowRelevanceFloor + 1, KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS);
2620
+ continue;
2621
+ }
2622
+ const freshness = knowledgeFreshness(candidate.record.quality.freshnessAt, nowMs);
2623
+ const qualityAdjustment = freshness === "current" ? 0.02 : freshness === "aging" ? 0.01 : 0;
2624
+ relevant.push({
2625
+ record: candidate.record,
2626
+ retrieval: {
2627
+ score: roundScore(Math.min(1, candidate.semanticScore + qualityAdjustment)),
2628
+ semanticScore: roundScore(candidate.semanticScore),
2629
+ matchType: candidate.matchType,
2630
+ vectorScore: candidate.vectorScore === null ? null : roundScore(candidate.vectorScore),
2631
+ keywordScore: candidate.keywordScore === null ? null : roundScore(candidate.keywordScore),
2632
+ relevanceSignals,
2633
+ freshness,
2634
+ qualityAdjustment,
2635
+ duplicateCount: 0,
2636
+ },
2637
+ });
2638
+ }
2639
+ relevant.sort(compareKnowledgeSearchResults);
2640
+
2641
+ const deduped: KnowledgeSearchResult[] = [];
2642
+ const byContent = new Map<string, number>();
2643
+ let asDuplicate = 0;
2644
+ for (const result of relevant) {
2645
+ const key = knowledgeTextualContentKey(result.record);
2646
+ const retainedIndex = byContent.get(key);
2647
+ if (retainedIndex === undefined) {
2648
+ byContent.set(key, deduped.length);
2649
+ deduped.push(result);
2650
+ continue;
2651
+ }
2652
+ asDuplicate += 1;
2653
+ const retained = deduped[retainedIndex]!;
2654
+ retained.retrieval.duplicateCount += 1;
2655
+ }
2656
+
2657
+ const forLimit = Math.max(0, deduped.length - requestedLimit);
2658
+ const bounded = deduped.slice(0, requestedLimit);
2659
+ let forResponseBudget = 0;
2660
+ let response = knowledgeSearchResponse({
2661
+ results: bounded,
2662
+ rankedCandidateCount: input.rankedCandidateCount,
2663
+ recheckedCandidateCount: input.candidates.length,
2664
+ belowRelevanceFloor,
2665
+ asDuplicate,
2666
+ forLimit,
2667
+ forResponseBudget,
2668
+ });
2669
+ while (knowledgeResponseBytes(response) > KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES) {
2670
+ if (bounded.length === 0) {
2671
+ throw new Error("knowledge search selection facts exceed the response budget");
2672
+ }
2673
+ bounded.pop();
2674
+ forResponseBudget += 1;
2675
+ response = knowledgeSearchResponse({
2676
+ results: bounded,
2677
+ rankedCandidateCount: input.rankedCandidateCount,
2678
+ recheckedCandidateCount: input.candidates.length,
2679
+ belowRelevanceFloor,
2680
+ asDuplicate,
2681
+ forLimit,
2682
+ forResponseBudget,
2683
+ });
2684
+ }
2685
+ return response;
2686
+ }
2687
+
2688
+ function compareKnowledgeSearchResults(
2689
+ left: KnowledgeSearchResult,
2690
+ right: KnowledgeSearchResult,
2691
+ ): number {
2692
+ return (
2693
+ right.retrieval.score - left.retrieval.score ||
2694
+ right.retrieval.semanticScore - left.retrieval.semanticScore ||
2695
+ (right.retrieval.vectorScore ?? 0) - (left.retrieval.vectorScore ?? 0) ||
2696
+ (right.retrieval.keywordScore ?? 0) - (left.retrieval.keywordScore ?? 0) ||
2697
+ (left.record.id === right.record.id ? 0 : left.record.id < right.record.id ? -1 : 1)
2698
+ );
2699
+ }
2700
+
2701
+ function knowledgeFreshness(value: string, nowMs: number): "current" | "aging" | "stale" {
2702
+ const freshnessMs = Date.parse(value);
2703
+ if (!Number.isFinite(freshnessMs)) return "stale";
2704
+ const ageDays = Math.max(0, (nowMs - freshnessMs) / 86_400_000);
2705
+ if (ageDays <= 90) return "current";
2706
+ return ageDays <= 365 ? "aging" : "stale";
2707
+ }
2708
+
2709
+ function knowledgeTextualContentKey(record: KnowledgeRecord): string {
2710
+ return createHash("sha256")
2711
+ .update("opengeni:knowledge-search-content:v1\0")
2712
+ .update(record.title)
2713
+ .update("\0")
2714
+ .update(
2715
+ JSON.stringify({
2716
+ body: record.content.body,
2717
+ summary: record.content.summary,
2718
+ topics: record.content.topics,
2719
+ }),
2720
+ )
2721
+ .digest("hex");
2722
+ }
2723
+
2724
+ function knowledgeSearchResponse(input: {
2725
+ results: KnowledgeSearchResult[];
2726
+ rankedCandidateCount: number;
2727
+ recheckedCandidateCount: number;
2728
+ belowRelevanceFloor: number;
2729
+ asDuplicate: number;
2730
+ forLimit: number;
2731
+ forResponseBudget: number;
2732
+ }): KnowledgeSearchResponse {
2733
+ const selection = {
2734
+ relevanceFloor: {
2735
+ policy: "any_signal" as const,
2736
+ vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE as 0.52,
2737
+ keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE as 0.01,
2738
+ },
2739
+ dedupe: { policy: "exact_textual_content" as const },
2740
+ candidates: {
2741
+ ranked: input.rankedCandidateCount,
2742
+ rechecked: input.recheckedCandidateCount,
2743
+ omittedOnRecheck: Math.max(0, input.rankedCandidateCount - input.recheckedCandidateCount),
2744
+ },
2745
+ omitted: {
2746
+ belowRelevanceFloor: input.belowRelevanceFloor,
2747
+ asDuplicate: input.asDuplicate,
2748
+ forLimit: input.forLimit,
2749
+ forResponseBudget: input.forResponseBudget,
2750
+ },
2751
+ budget: {
2752
+ maxResults: KNOWLEDGE_SEARCH_MAX_RESULTS as 50,
2753
+ maxResponseBytes: KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES as 65_536,
2754
+ responseBytes: 0,
2755
+ tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
2756
+ estimatedTokens: 0,
2757
+ maxEstimatedTokens: Math.ceil(
2758
+ KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2759
+ ) as 16_384,
2760
+ },
2761
+ };
2762
+ const response: KnowledgeSearchResponse = {
2763
+ results: [...input.results],
2764
+ selection,
1919
2765
  };
2766
+ // The counters themselves contribute a few bytes. Iterate to a fixed point so
2767
+ // responseBytes describes the actual serialized response, not an approximation.
2768
+ for (let index = 0; index < 8; index += 1) {
2769
+ const responseBytes = knowledgeResponseBytes(response);
2770
+ const estimatedTokens = Math.ceil(
2771
+ responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2772
+ );
2773
+ if (
2774
+ response.selection.budget.responseBytes === responseBytes &&
2775
+ response.selection.budget.estimatedTokens === estimatedTokens
2776
+ ) {
2777
+ break;
2778
+ }
2779
+ response.selection.budget.responseBytes = responseBytes;
2780
+ response.selection.budget.estimatedTokens = estimatedTokens;
2781
+ }
2782
+ return response;
2783
+ }
2784
+
2785
+ function knowledgeResponseBytes(response: KnowledgeSearchResponse): number {
2786
+ return Buffer.byteLength(JSON.stringify(response), "utf8");
2787
+ }
2788
+
2789
+ type KnowledgeBrowseEntry = {
2790
+ record: KnowledgeRecord;
2791
+ cursorAfter: string;
2792
+ };
2793
+
2794
+ /**
2795
+ * Bound a browse page without skipping records. Tail records omitted for the
2796
+ * response budget remain behind the returned cursor. If one complete record is
2797
+ * itself too large, return a deterministic discovery projection; knowledge_get
2798
+ * remains the freshly authorized full-record path.
2799
+ */
2800
+ export function selectKnowledgeBrowseRecords(input: {
2801
+ entries: KnowledgeBrowseEntry[];
2802
+ hasMoreAfterEntries: boolean;
2803
+ }): KnowledgeBrowseResponse {
2804
+ const entries = input.entries.slice(0, KNOWLEDGE_BROWSE_MAX_LIMIT);
2805
+ const selected = entries.map((entry) => entry.record);
2806
+ let omittedForResponseBudget = 0;
2807
+ let compactedRecordCount = 0;
2808
+ let response = knowledgeBrowseResponse({
2809
+ records: selected,
2810
+ nextCursor: input.hasMoreAfterEntries ? (entries.at(-1)?.cursorAfter ?? null) : null,
2811
+ hasMore: input.hasMoreAfterEntries,
2812
+ omittedForResponseBudget,
2813
+ compactedRecordCount,
2814
+ });
2815
+ while (
2816
+ selected.length > 1 &&
2817
+ knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
2818
+ ) {
2819
+ selected.pop();
2820
+ omittedForResponseBudget += 1;
2821
+ response = knowledgeBrowseResponse({
2822
+ records: selected,
2823
+ nextCursor: entries[selected.length - 1]?.cursorAfter ?? null,
2824
+ hasMore: true,
2825
+ omittedForResponseBudget,
2826
+ compactedRecordCount,
2827
+ });
2828
+ }
2829
+ if (
2830
+ selected.length === 1 &&
2831
+ knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
2832
+ ) {
2833
+ selected[0] = compactKnowledgeBrowseRecord(selected[0]!);
2834
+ compactedRecordCount = 1;
2835
+ response = knowledgeBrowseResponse({
2836
+ records: selected,
2837
+ nextCursor:
2838
+ omittedForResponseBudget > 0 || input.hasMoreAfterEntries
2839
+ ? (entries[0]?.cursorAfter ?? null)
2840
+ : null,
2841
+ hasMore: omittedForResponseBudget > 0 || input.hasMoreAfterEntries,
2842
+ omittedForResponseBudget,
2843
+ compactedRecordCount,
2844
+ });
2845
+ }
2846
+ if (knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES) {
2847
+ throw new Error("knowledge browse discovery projection exceeds the response budget");
2848
+ }
2849
+ return response;
2850
+ }
2851
+
2852
+ function compactKnowledgeBrowseRecord(record: KnowledgeRecord): KnowledgeRecord {
2853
+ const fields = new Set<KnowledgeRecord["projection"]["fields"][number]>([
2854
+ ...record.projection.fields,
2855
+ "content.body",
2856
+ "content.summary",
2857
+ "content.topics",
2858
+ "content.metadata",
2859
+ "provenance.source.uri",
2860
+ "provenance.source.externalId",
2861
+ "provenance.source.title",
2862
+ "provenance.source.author",
2863
+ "provenance.source.version",
2864
+ "provenance.citation",
2865
+ ]);
2866
+ return {
2867
+ ...record,
2868
+ content: {
2869
+ format: "markdown",
2870
+ body: null,
2871
+ summary: null,
2872
+ topics: [],
2873
+ metadata: {},
2874
+ },
2875
+ provenance: {
2876
+ ...record.provenance,
2877
+ source: {
2878
+ ...record.provenance.source,
2879
+ uri: null,
2880
+ externalId: null,
2881
+ title: null,
2882
+ author: null,
2883
+ version: null,
2884
+ },
2885
+ citation: null,
2886
+ },
2887
+ links: record.links.filter((link) => link.target.kind === "knowledge"),
2888
+ projection: {
2889
+ truncated: true,
2890
+ fields: [...fields].sort(),
2891
+ },
2892
+ };
2893
+ }
2894
+
2895
+ function knowledgeBrowseResponse(input: {
2896
+ records: KnowledgeRecord[];
2897
+ nextCursor: string | null;
2898
+ hasMore: boolean;
2899
+ omittedForResponseBudget: number;
2900
+ compactedRecordCount: number;
2901
+ }): KnowledgeBrowseResponse {
2902
+ const response: KnowledgeBrowseResponse = {
2903
+ records: [...input.records],
2904
+ nextCursor: input.nextCursor,
2905
+ hasMore: input.hasMore,
2906
+ selection: {
2907
+ omitted: { forResponseBudget: input.omittedForResponseBudget },
2908
+ compactedRecordCount: input.compactedRecordCount,
2909
+ budget: {
2910
+ maxResults: KNOWLEDGE_BROWSE_MAX_LIMIT as 50,
2911
+ maxResponseBytes: KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES as 65_536,
2912
+ responseBytes: 0,
2913
+ tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
2914
+ estimatedTokens: 0,
2915
+ maxEstimatedTokens: Math.ceil(
2916
+ KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2917
+ ) as 16_384,
2918
+ },
2919
+ },
2920
+ };
2921
+ for (let index = 0; index < 8; index += 1) {
2922
+ const responseBytes = knowledgeBrowseResponseBytes(response);
2923
+ const estimatedTokens = Math.ceil(
2924
+ responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
2925
+ );
2926
+ if (
2927
+ response.selection.budget.responseBytes === responseBytes &&
2928
+ response.selection.budget.estimatedTokens === estimatedTokens
2929
+ ) {
2930
+ break;
2931
+ }
2932
+ response.selection.budget.responseBytes = responseBytes;
2933
+ response.selection.budget.estimatedTokens = estimatedTokens;
2934
+ }
2935
+ return response;
2936
+ }
2937
+
2938
+ function knowledgeBrowseResponseBytes(response: KnowledgeBrowseResponse): number {
2939
+ return Buffer.byteLength(JSON.stringify(response), "utf8");
1920
2940
  }
1921
2941
 
1922
2942
  /** Fetch one stable Knowledge record with a fresh authorization check. */
@@ -1926,12 +2946,20 @@ export async function getEffectiveKnowledgeRecord(
1926
2946
  accountId: string;
1927
2947
  workspaceId: string;
1928
2948
  initiatingSubjectId: string;
2949
+ surface?: "human" | "agent" | undefined;
1929
2950
  id: string;
2951
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
1930
2952
  },
1931
2953
  ): Promise<KnowledgeRecord | null> {
1932
2954
  const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1933
2955
  const target = parseKnowledgeRecordId(input.id);
1934
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
2956
+ const access = await resolveEffectiveDocumentAccess(db, {
2957
+ accountId: input.accountId,
2958
+ workspaceId: input.workspaceId,
2959
+ initiatingSubjectId,
2960
+ surface: input.surface ?? "agent",
2961
+ agentAuthority: input.agentAuthority,
2962
+ });
1935
2963
  return await withDocumentAccountRls(
1936
2964
  db,
1937
2965
  input.accountId,
@@ -1943,8 +2971,17 @@ export async function getEffectiveKnowledgeRecord(
1943
2971
  .select({
1944
2972
  document: schema.documents,
1945
2973
  citation: googleDriveCitationProjection(input.workspaceId, access),
2974
+ firstChunkId: schema.documentChunks.id,
1946
2975
  })
1947
2976
  .from(schema.documents)
2977
+ .leftJoin(
2978
+ schema.documentChunks,
2979
+ and(
2980
+ eq(schema.documentChunks.accountId, schema.documents.accountId),
2981
+ eq(schema.documentChunks.documentId, schema.documents.id),
2982
+ eq(schema.documentChunks.chunkIndex, 0),
2983
+ ),
2984
+ )
1948
2985
  .where(
1949
2986
  and(
1950
2987
  eq(schema.documents.accountId, input.accountId),
@@ -1954,13 +2991,15 @@ export async function getEffectiveKnowledgeRecord(
1954
2991
  ),
1955
2992
  )
1956
2993
  .limit(1);
1957
- return row ? knowledgeDocumentRecord(row.document, row.citation) : null;
2994
+ return row ? knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId) : null;
1958
2995
  }
1959
2996
  const [row] = await scopedDb
1960
2997
  .select({
1961
2998
  chunk: schema.documentChunks,
1962
2999
  document: schema.documents,
1963
3000
  citation: googleDriveCitationProjection(input.workspaceId, access),
3001
+ previousChunkId: knowledgePreviousChunkIdProjection(),
3002
+ nextChunkId: knowledgeNextChunkIdProjection(),
1964
3003
  })
1965
3004
  .from(schema.documentChunks)
1966
3005
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
@@ -1974,7 +3013,12 @@ export async function getEffectiveKnowledgeRecord(
1974
3013
  ),
1975
3014
  )
1976
3015
  .limit(1);
1977
- return row ? knowledgeChunkRecord(row.document, row.chunk, row.citation) : null;
3016
+ return row
3017
+ ? knowledgeChunkRecord(row.document, row.chunk, row.citation, {
3018
+ previousChunkId: row.previousChunkId,
3019
+ nextChunkId: row.nextChunkId,
3020
+ })
3021
+ : null;
1978
3022
  },
1979
3023
  );
1980
3024
  }
@@ -2005,7 +3049,7 @@ export async function browseEffectiveKnowledge(
2005
3049
  if (parent && (topic || sourceKinds.length > 0)) {
2006
3050
  throw new Error("knowledge browse document contents do not accept topic/source filters");
2007
3051
  }
2008
- const cursorScope = {
3052
+ const cursorScope: KnowledgeBrowseCursorScope = {
2009
3053
  accountId: input.accountId,
2010
3054
  workspaceId: input.workspaceId,
2011
3055
  initiatingSubjectId,
@@ -2013,11 +3057,15 @@ export async function browseEffectiveKnowledge(
2013
3057
  topic,
2014
3058
  sourceKinds,
2015
3059
  };
2016
- const after = input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
2017
- if (parent && after > 2_147_483_648n) {
2018
- throw new Error("invalid knowledge browse cursor");
2019
- }
2020
- const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
3060
+ const topLevelAfter =
3061
+ !parent && input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
3062
+ const access = await resolveEffectiveDocumentAccess(db, {
3063
+ accountId: input.accountId,
3064
+ workspaceId: input.workspaceId,
3065
+ initiatingSubjectId,
3066
+ surface: input.surface ?? "agent",
3067
+ agentAuthority: input.agentAuthority,
3068
+ });
2021
3069
  return await withDocumentAccountRls(
2022
3070
  db,
2023
3071
  input.accountId,
@@ -2026,7 +3074,7 @@ export async function browseEffectiveKnowledge(
2026
3074
  async (scopedDb) => {
2027
3075
  if (parent) {
2028
3076
  const [authorizedParent] = await scopedDb
2029
- .select({ id: schema.documents.id })
3077
+ .select({ id: schema.documents.id, indexSequence: schema.documents.indexSequence })
2030
3078
  .from(schema.documents)
2031
3079
  .where(
2032
3080
  and(
@@ -2037,12 +3085,29 @@ export async function browseEffectiveKnowledge(
2037
3085
  ),
2038
3086
  )
2039
3087
  .limit(1);
2040
- if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
3088
+ if (!authorizedParent) {
3089
+ return selectKnowledgeBrowseRecords({ entries: [], hasMoreAfterEntries: false });
3090
+ }
3091
+ if (authorizedParent.indexSequence === null) {
3092
+ throw new Error("ready knowledge document is missing its index revision");
3093
+ }
3094
+ const parentCursorScope: KnowledgeBrowseCursorScope = {
3095
+ ...cursorScope,
3096
+ parentRevision: authorizedParent.indexSequence.toString(),
3097
+ };
3098
+ const after = input.cursor
3099
+ ? decodeKnowledgeBrowseCursor(input.cursor, parentCursorScope)
3100
+ : 0n;
3101
+ if (after > 2_147_483_648n) {
3102
+ throw new Error("invalid knowledge browse cursor");
3103
+ }
2041
3104
  const rows = await scopedDb
2042
3105
  .select({
2043
3106
  chunk: schema.documentChunks,
2044
3107
  document: schema.documents,
2045
3108
  citation: googleDriveCitationProjection(input.workspaceId, access),
3109
+ previousChunkId: knowledgePreviousChunkIdProjection(),
3110
+ nextChunkId: knowledgeNextChunkIdProjection(),
2046
3111
  })
2047
3112
  .from(schema.documentChunks)
2048
3113
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
@@ -2052,6 +3117,7 @@ export async function browseEffectiveKnowledge(
2052
3117
  eq(schema.documentChunks.documentId, parent.id),
2053
3118
  gt(schema.documentChunks.chunkIndex, Number(after) - 1),
2054
3119
  eq(schema.documents.status, "ready"),
3120
+ eq(schema.documents.indexSequence, authorizedParent.indexSequence),
2055
3121
  ...documentAccessConditions(input.workspaceId, access),
2056
3122
  ),
2057
3123
  )
@@ -2059,22 +3125,26 @@ export async function browseEffectiveKnowledge(
2059
3125
  .limit(limit + 1);
2060
3126
  const hasMore = rows.length > limit;
2061
3127
  const page = rows.slice(0, limit);
2062
- const last = page.at(-1)?.chunk.chunkIndex;
2063
- return {
2064
- records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk, row.citation)),
2065
- nextCursor:
2066
- hasMore && last !== undefined
2067
- ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last + 1))
2068
- : null,
2069
- hasMore,
2070
- };
3128
+ return selectKnowledgeBrowseRecords({
3129
+ entries: page.map((row) => ({
3130
+ record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
3131
+ previousChunkId: row.previousChunkId,
3132
+ nextChunkId: row.nextChunkId,
3133
+ }),
3134
+ cursorAfter: encodeKnowledgeBrowseCursor(
3135
+ parentCursorScope,
3136
+ BigInt(row.chunk.chunkIndex + 1),
3137
+ ),
3138
+ })),
3139
+ hasMoreAfterEntries: hasMore,
3140
+ });
2071
3141
  }
2072
3142
 
2073
3143
  const conditions: SQL[] = [
2074
3144
  eq(schema.documents.accountId, input.accountId),
2075
3145
  eq(schema.documents.status, "ready"),
2076
3146
  isNotNull(schema.documents.indexSequence),
2077
- gt(schema.documents.indexSequence, after),
3147
+ gt(schema.documents.indexSequence, topLevelAfter),
2078
3148
  ...documentAccessConditions(input.workspaceId, access),
2079
3149
  ];
2080
3150
  if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
@@ -2084,22 +3154,34 @@ export async function browseEffectiveKnowledge(
2084
3154
  .select({
2085
3155
  document: schema.documents,
2086
3156
  citation: googleDriveCitationProjection(input.workspaceId, access),
3157
+ firstChunkId: schema.documentChunks.id,
2087
3158
  })
2088
3159
  .from(schema.documents)
3160
+ .leftJoin(
3161
+ schema.documentChunks,
3162
+ and(
3163
+ eq(schema.documentChunks.accountId, schema.documents.accountId),
3164
+ eq(schema.documentChunks.documentId, schema.documents.id),
3165
+ eq(schema.documentChunks.chunkIndex, 0),
3166
+ ),
3167
+ )
2089
3168
  .where(and(...conditions))
2090
3169
  .orderBy(asc(schema.documents.indexSequence))
2091
3170
  .limit(limit + 1);
2092
3171
  const hasMore = rows.length > limit;
2093
3172
  const page = rows.slice(0, limit);
2094
- const last = page.at(-1)?.document.indexSequence;
2095
- return {
2096
- records: page.map((row) => knowledgeDocumentRecord(row.document, row.citation)),
2097
- nextCursor:
2098
- hasMore && last !== undefined && last !== null
2099
- ? encodeKnowledgeBrowseCursor(cursorScope, last)
2100
- : null,
2101
- hasMore,
2102
- };
3173
+ return selectKnowledgeBrowseRecords({
3174
+ entries: page.map((row) => {
3175
+ if (row.document.indexSequence === null) {
3176
+ throw new Error("ready knowledge document is missing its index revision");
3177
+ }
3178
+ return {
3179
+ record: knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId),
3180
+ cursorAfter: encodeKnowledgeBrowseCursor(cursorScope, row.document.indexSequence),
3181
+ };
3182
+ }),
3183
+ hasMoreAfterEntries: hasMore,
3184
+ });
2103
3185
  },
2104
3186
  );
2105
3187
  }
@@ -2109,6 +3191,7 @@ type KnowledgeBrowseCursorScope = {
2109
3191
  workspaceId: string;
2110
3192
  initiatingSubjectId: string;
2111
3193
  parentId: string | null;
3194
+ parentRevision?: string | null | undefined;
2112
3195
  topic: string | null;
2113
3196
  sourceKinds: readonly string[];
2114
3197
  };
@@ -2118,8 +3201,9 @@ export function encodeKnowledgeBrowseCursor(
2118
3201
  position: bigint,
2119
3202
  ): string {
2120
3203
  if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
3204
+ const version = knowledgeBrowseCursorVersion(scope);
2121
3205
  return Buffer.from(
2122
- JSON.stringify({ v: 1, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
3206
+ JSON.stringify({ v: version, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
2123
3207
  "utf8",
2124
3208
  ).toString("base64url");
2125
3209
  }
@@ -2135,9 +3219,10 @@ export function decodeKnowledgeBrowseCursor(
2135
3219
  const bytes = Buffer.from(value, "base64url");
2136
3220
  if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
2137
3221
  const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
3222
+ const expectedVersion = knowledgeBrowseCursorVersion(scope);
2138
3223
  if (
2139
3224
  Object.keys(parsed).sort().join(",") !== "q,s,v" ||
2140
- parsed.v !== 1 ||
3225
+ parsed.v !== expectedVersion ||
2141
3226
  typeof parsed.s !== "string" ||
2142
3227
  typeof parsed.q !== "string" ||
2143
3228
  !/^(0|[1-9][0-9]*)$/.test(parsed.q)
@@ -2162,8 +3247,9 @@ export function decodeKnowledgeBrowseCursor(
2162
3247
  }
2163
3248
 
2164
3249
  function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
2165
- return createHash("sha256")
2166
- .update("opengeni:knowledge-browse-cursor:v1\0")
3250
+ const version = knowledgeBrowseCursorVersion(scope);
3251
+ const hash = createHash("sha256")
3252
+ .update(`opengeni:knowledge-browse-cursor:v${version}\0`)
2167
3253
  .update(scope.accountId)
2168
3254
  .update("\0")
2169
3255
  .update(scope.workspaceId)
@@ -2171,13 +3257,23 @@ function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
2171
3257
  .update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
2172
3258
  .update("\0")
2173
3259
  .update(scope.parentId ?? "")
2174
- .update("\0")
3260
+ .update("\0");
3261
+ if (version === 2) hash.update(scope.parentRevision!).update("\0");
3262
+ return hash
2175
3263
  .update(scope.topic ?? "")
2176
3264
  .update("\0")
2177
3265
  .update([...scope.sourceKinds].sort().join("\0"))
2178
3266
  .digest("hex");
2179
3267
  }
2180
3268
 
3269
+ function knowledgeBrowseCursorVersion(scope: KnowledgeBrowseCursorScope): 1 | 2 {
3270
+ if (!scope.parentId) return 1;
3271
+ if (!scope.parentRevision || !/^[1-9][0-9]*$/.test(scope.parentRevision)) {
3272
+ throw new Error("knowledge browse parent cursor requires an exact document revision");
3273
+ }
3274
+ return 2;
3275
+ }
3276
+
2181
3277
  function parseKnowledgeRecordId(value: string): {
2182
3278
  kind: "document" | "document_chunk";
2183
3279
  id: string;
@@ -2193,6 +3289,7 @@ function parseKnowledgeRecordId(value: string): {
2193
3289
  function knowledgeDocumentRecord(
2194
3290
  document: typeof schema.documents.$inferSelect,
2195
3291
  citation: unknown = null,
3292
+ firstChunkId: string | null = null,
2196
3293
  ): KnowledgeRecord {
2197
3294
  if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2198
3295
  const projected = projectKnowledgeRecord({
@@ -2216,7 +3313,20 @@ function knowledgeDocumentRecord(
2216
3313
  },
2217
3314
  lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2218
3315
  quality: knowledgeQuality(document),
2219
- links: knowledgeSourceLinks(projected.source.uri),
3316
+ links: [
3317
+ ...(firstChunkId
3318
+ ? [
3319
+ {
3320
+ relation: "contents" as const,
3321
+ target: {
3322
+ kind: "knowledge" as const,
3323
+ id: `document_chunk:${firstChunkId}` as const,
3324
+ },
3325
+ },
3326
+ ]
3327
+ : []),
3328
+ ...knowledgeSourceLinks(projected.source.uri),
3329
+ ],
2220
3330
  projection: projected.projection,
2221
3331
  };
2222
3332
  }
@@ -2225,6 +3335,10 @@ function knowledgeChunkRecord(
2225
3335
  document: typeof schema.documents.$inferSelect,
2226
3336
  chunk: typeof schema.documentChunks.$inferSelect,
2227
3337
  citation: unknown = null,
3338
+ traversal: {
3339
+ previousChunkId: string | null;
3340
+ nextChunkId: string | null;
3341
+ } = { previousChunkId: null, nextChunkId: null },
2228
3342
  ): KnowledgeRecord {
2229
3343
  if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2230
3344
  const projected = projectKnowledgeRecord({
@@ -2249,7 +3363,32 @@ function knowledgeChunkRecord(
2249
3363
  lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2250
3364
  quality: knowledgeQuality(document),
2251
3365
  links: [
2252
- { relation: "parent", target: { kind: "knowledge", id: `document:${document.id}` } },
3366
+ {
3367
+ relation: "parent",
3368
+ target: { kind: "knowledge", id: `document:${document.id}` },
3369
+ },
3370
+ ...(traversal.previousChunkId
3371
+ ? [
3372
+ {
3373
+ relation: "previous" as const,
3374
+ target: {
3375
+ kind: "knowledge" as const,
3376
+ id: `document_chunk:${traversal.previousChunkId}` as const,
3377
+ },
3378
+ },
3379
+ ]
3380
+ : []),
3381
+ ...(traversal.nextChunkId
3382
+ ? [
3383
+ {
3384
+ relation: "next" as const,
3385
+ target: {
3386
+ kind: "knowledge" as const,
3387
+ id: `document_chunk:${traversal.nextChunkId}` as const,
3388
+ },
3389
+ },
3390
+ ]
3391
+ : []),
2253
3392
  ...knowledgeSourceLinks(projected.source.uri),
2254
3393
  ],
2255
3394
  projection: projected.projection,
@@ -2285,6 +3424,35 @@ function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"
2285
3424
  return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
2286
3425
  }
2287
3426
 
3427
+ /**
3428
+ * These structural targets are selected inside the same authorization-scoped
3429
+ * transaction as their owning record. Only opaque ids are projected; titles,
3430
+ * source fields, and content require a subsequent freshly authorized get.
3431
+ */
3432
+ function knowledgePreviousChunkIdProjection(): SQL<string | null> {
3433
+ return sql<string | null>`(
3434
+ select knowledge_previous_chunk.id
3435
+ from document_chunks knowledge_previous_chunk
3436
+ where knowledge_previous_chunk.account_id = ${schema.documentChunks.accountId}
3437
+ and knowledge_previous_chunk.document_id = ${schema.documentChunks.documentId}
3438
+ and knowledge_previous_chunk.chunk_index < ${schema.documentChunks.chunkIndex}
3439
+ order by knowledge_previous_chunk.chunk_index desc
3440
+ limit 1
3441
+ )`;
3442
+ }
3443
+
3444
+ function knowledgeNextChunkIdProjection(): SQL<string | null> {
3445
+ return sql<string | null>`(
3446
+ select knowledge_next_chunk.id
3447
+ from document_chunks knowledge_next_chunk
3448
+ where knowledge_next_chunk.account_id = ${schema.documentChunks.accountId}
3449
+ and knowledge_next_chunk.document_id = ${schema.documentChunks.documentId}
3450
+ and knowledge_next_chunk.chunk_index > ${schema.documentChunks.chunkIndex}
3451
+ order by knowledge_next_chunk.chunk_index asc
3452
+ limit 1
3453
+ )`;
3454
+ }
3455
+
2288
3456
  async function vectorSearchDocuments(
2289
3457
  db: Database,
2290
3458
  input: DocumentSearchInput,
@@ -2329,7 +3497,7 @@ async function vectorSearchDocuments(
2329
3497
  .from(schema.documentChunks)
2330
3498
  .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
2331
3499
  .where(and(...documentSearchConditions(input, services.embedder.model)))
2332
- .orderBy(distance)
3500
+ .orderBy(distance, asc(schema.documentChunks.id))
2333
3501
  .limit(limit),
2334
3502
  );
2335
3503
  return rows.map((row) => ({
@@ -2385,7 +3553,7 @@ async function keywordSearchDocuments(
2385
3553
  sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
2386
3554
  ),
2387
3555
  )
2388
- .orderBy(desc(rank))
3556
+ .orderBy(desc(rank), asc(schema.documentChunks.id))
2389
3557
  .limit(limit),
2390
3558
  );
2391
3559
  return rows.map((row) => ({
@@ -2535,6 +3703,33 @@ async function assertDocumentAccountWorkspace(
2535
3703
  return context;
2536
3704
  }
2537
3705
 
3706
+ export async function resolveEffectiveDocumentAccess(
3707
+ _db: Database,
3708
+ input: {
3709
+ accountId: string;
3710
+ workspaceId: string;
3711
+ initiatingSubjectId: string;
3712
+ surface: "human" | "agent";
3713
+ agentAuthority?: AgentDocumentAuthorityContext | undefined;
3714
+ },
3715
+ ): Promise<DocumentAccessFilter> {
3716
+ if (input.surface === "human") {
3717
+ return { viewerSubjectId: input.initiatingSubjectId };
3718
+ }
3719
+ return {
3720
+ agentOnly: true,
3721
+ viewerSubjectId: input.initiatingSubjectId,
3722
+ authorizedPersonalAttempt: input.agentAuthority
3723
+ ? {
3724
+ accountId: input.accountId,
3725
+ workspaceId: input.workspaceId,
3726
+ sessionId: input.agentAuthority.sessionId,
3727
+ attemptId: input.agentAuthority.attemptId,
3728
+ }
3729
+ : undefined,
3730
+ };
3731
+ }
3732
+
2538
3733
  /**
2539
3734
  * Visibility/agent scoping shared by every document read path. Fail-closed:
2540
3735
  * with no filter supplied, private documents are invisible.
@@ -2549,11 +3744,33 @@ function documentAccessConditions(
2549
3744
  eq(schema.documents.authorityWorkspaceId, workspaceId),
2550
3745
  );
2551
3746
  const viewer = cleanString(access?.viewerSubjectId ?? null);
3747
+ const personalAttempt = access?.authorizedPersonalAttempt;
3748
+ const authorizedPersonal = personalAttempt
3749
+ ? sql`${schema.documents.id} IN (
3750
+ SELECT resolve_session_attempt_personal_document_reads(
3751
+ ${personalAttempt.accountId}::uuid,
3752
+ ${personalAttempt.workspaceId}::uuid,
3753
+ ${personalAttempt.sessionId}::uuid,
3754
+ ${personalAttempt.attemptId}::uuid
3755
+ )
3756
+ )`
3757
+ : sql`false`;
2552
3758
  const personal = viewer
2553
3759
  ? and(
2554
3760
  eq(schema.documents.authorityKind, "personal"),
2555
- eq(schema.documents.authorityWorkspaceId, workspaceId),
2556
3761
  eq(schema.documents.authoritySubjectId, viewer),
3762
+ access?.agentOnly
3763
+ ? (or(
3764
+ and(
3765
+ isNull(schema.documents.authorityId),
3766
+ eq(schema.documents.authorityWorkspaceId, workspaceId),
3767
+ ),
3768
+ and(isNotNull(schema.documents.authorityId), authorizedPersonal),
3769
+ ) ?? authorizedPersonal)
3770
+ : (or(
3771
+ eq(schema.documents.authorityWorkspaceId, workspaceId),
3772
+ isNull(schema.documents.authorityWorkspaceId),
3773
+ ) ?? eq(schema.documents.authorityWorkspaceId, workspaceId)),
2557
3774
  )
2558
3775
  : undefined;
2559
3776
  const authority = viewer
@@ -2575,13 +3792,22 @@ function documentAccessConditions(
2575
3792
  function documentMatchesAccess(
2576
3793
  document: Pick<
2577
3794
  DocumentAccessRecord,
2578
- "authorityKind" | "authorityWorkspaceId" | "authoritySubjectId" | "agentAccess"
3795
+ | "id"
3796
+ | "authorityId"
3797
+ | "authorityKind"
3798
+ | "authorityWorkspaceId"
3799
+ | "authoritySubjectId"
3800
+ | "agentAccess"
2579
3801
  >,
2580
3802
  workspaceId: string,
2581
3803
  access: DocumentAccessFilter | undefined,
2582
3804
  ): boolean {
2583
3805
  if (access?.agentOnly) {
2584
- return document.agentAccess && canViewDocument(document, access.viewerSubjectId, workspaceId);
3806
+ return (
3807
+ document.agentAccess &&
3808
+ (document.authorityKind !== "personal" || document.authorityId === null) &&
3809
+ canViewDocument(document, access.viewerSubjectId, workspaceId)
3810
+ );
2585
3811
  }
2586
3812
  return canViewDocument(document, access?.viewerSubjectId, workspaceId);
2587
3813
  }
@@ -2599,9 +3825,9 @@ function canonicalDocumentAuthoritySubject(value: unknown): string | undefined {
2599
3825
  /**
2600
3826
  * Whether a single already-fetched document is readable in this workspace.
2601
3827
  *
2602
- * Keep this compatibility predicate as strict as SQL/RLS: personal authority
2603
- * remains anchored to its originating workspace, and unknown or incomplete
2604
- * authority tuples deny instead of falling through as workspace-visible.
3828
+ * Keep this compatibility predicate as strict as SQL/RLS: legacy personal
3829
+ * authority remains origin-workspace anchored while activated personal
3830
+ * authority has a null workspace and follows the exact owner within the org.
2605
3831
  */
2606
3832
  export function canViewDocument(
2607
3833
  document: Pick<DocumentAccessRecord, "authorityKind" | "authoritySubjectId"> & {
@@ -2614,21 +3840,31 @@ export function canViewDocument(
2614
3840
  return document.authorityWorkspaceId === null && document.authoritySubjectId === null;
2615
3841
  }
2616
3842
  const normalizedWorkspaceId = cleanString(workspaceId ?? null);
2617
- if (!normalizedWorkspaceId || document.authorityWorkspaceId !== normalizedWorkspaceId) {
3843
+ if (!normalizedWorkspaceId) {
2618
3844
  return false;
2619
3845
  }
2620
3846
  if (document.authorityKind === "workspace") {
2621
- return document.authoritySubjectId === null;
3847
+ return (
3848
+ document.authorityWorkspaceId === normalizedWorkspaceId &&
3849
+ document.authoritySubjectId === null
3850
+ );
2622
3851
  }
2623
3852
  if (document.authorityKind === "personal") {
2624
3853
  const authoritySubjectId = canonicalDocumentAuthoritySubject(document.authoritySubjectId);
2625
3854
  const viewer = canonicalDocumentAuthoritySubject(viewerSubjectId);
2626
- return !!authoritySubjectId && authoritySubjectId === viewer;
3855
+ return (
3856
+ !!authoritySubjectId &&
3857
+ authoritySubjectId === viewer &&
3858
+ (document.authorityWorkspaceId === null ||
3859
+ document.authorityWorkspaceId === normalizedWorkspaceId)
3860
+ );
2627
3861
  }
2628
3862
  return false;
2629
3863
  }
2630
3864
 
2631
3865
  type DocumentAccessRecord = {
3866
+ id: string;
3867
+ authorityId: string | null;
2632
3868
  authorityKind: string;
2633
3869
  authorityWorkspaceId: string | null;
2634
3870
  authoritySubjectId: string | null;
@@ -2765,7 +4001,8 @@ function mergeDocumentSearchRows(
2765
4001
  right.score - left.score ||
2766
4002
  (right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
2767
4003
  (right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
2768
- left.chunkIndex - right.chunkIndex,
4004
+ left.chunkIndex - right.chunkIndex ||
4005
+ (left.chunkId === right.chunkId ? 0 : left.chunkId < right.chunkId ? -1 : 1),
2769
4006
  );
2770
4007
  }
2771
4008
 
@@ -3021,6 +4258,7 @@ function mapDocument(row: typeof schema.documents.$inferSelect): Document {
3021
4258
  authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
3022
4259
  authorityWorkspaceId: row.authorityWorkspaceId,
3023
4260
  authoritySubjectId: row.authoritySubjectId,
4261
+ authorityId: row.authorityId,
3024
4262
  visibility: normalizeDocumentVisibility(row.visibility),
3025
4263
  createdBy: row.createdBy,
3026
4264
  agentAccess: row.agentAccess,