@opengeni/documents 0.5.41 → 0.7.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +258 -5
- package/dist/index.js +991 -125
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/src/index.ts +1367 -129
package/src/index.ts
CHANGED
|
@@ -6,6 +6,8 @@ import {
|
|
|
6
6
|
CreateDocumentBaseRequest,
|
|
7
7
|
Document,
|
|
8
8
|
DocumentAuthorityKind,
|
|
9
|
+
DocumentAuthorityReclassification,
|
|
10
|
+
ListDocumentAuthorityReclassificationsResponse,
|
|
9
11
|
DocumentBase,
|
|
10
12
|
DocumentCuration,
|
|
11
13
|
DocumentCurationStatus,
|
|
@@ -17,12 +19,25 @@ import {
|
|
|
17
19
|
IndexedDocumentSummary,
|
|
18
20
|
KnowledgeBrowseResponse,
|
|
19
21
|
KnowledgeRecord,
|
|
22
|
+
KnowledgeSearchResult,
|
|
20
23
|
KnowledgeSearchResponse,
|
|
21
24
|
KnowledgeSourceKind,
|
|
22
25
|
ListIndexedDocumentsResponse,
|
|
26
|
+
type ReclassifyDocumentAuthorityRequest,
|
|
27
|
+
type RunDocumentDefaultCollectionBackfillRequest,
|
|
28
|
+
DocumentDefaultCollectionBackfill,
|
|
29
|
+
DocumentDefaultCollectionBackfillAudit,
|
|
30
|
+
DocumentDefaultCollectionBackfillOperationAudit,
|
|
31
|
+
DocumentDefaultCollectionBackfillReceiptAudit,
|
|
32
|
+
DocumentDefaultCollectionBackfillRunAudit,
|
|
33
|
+
ListDocumentDefaultCollectionBackfillRunsResponse,
|
|
34
|
+
ListOrganizationDocumentAuthorityReclassificationsResponse,
|
|
35
|
+
OrganizationDocumentAuthorityReclassification,
|
|
23
36
|
} from "@opengeni/contracts";
|
|
24
37
|
import {
|
|
38
|
+
createPersonalDocumentAuthority,
|
|
25
39
|
getFilesForSubject,
|
|
40
|
+
resolveDocumentOriginalFileForSubject,
|
|
26
41
|
rlsContextForWorkspace,
|
|
27
42
|
setSubjectRlsContext,
|
|
28
43
|
withRlsContext,
|
|
@@ -31,11 +46,21 @@ import {
|
|
|
31
46
|
type Database,
|
|
32
47
|
} from "@opengeni/db";
|
|
33
48
|
import * as schema from "@opengeni/db/schema";
|
|
34
|
-
import type
|
|
35
|
-
import { createHash } from "node:crypto";
|
|
36
|
-
import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
|
|
49
|
+
import { retryWhileMissing, type ObjectStorage } from "@opengeni/storage";
|
|
50
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
51
|
+
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from "drizzle-orm";
|
|
37
52
|
import type OpenAI from "openai";
|
|
38
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS,
|
|
55
|
+
KNOWLEDGE_BROWSE_MAX_LIMIT,
|
|
56
|
+
KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES,
|
|
57
|
+
KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES,
|
|
58
|
+
KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
|
|
59
|
+
KNOWLEDGE_SEARCH_MAX_RESULTS,
|
|
60
|
+
KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
|
|
61
|
+
KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
|
|
62
|
+
KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
63
|
+
} from "@opengeni/contracts";
|
|
39
64
|
import { projectKnowledgeRecord } from "./knowledge-projection";
|
|
40
65
|
|
|
41
66
|
export { projectKnowledgeRecord } from "./knowledge-projection";
|
|
@@ -138,6 +163,15 @@ export type DocumentAccessFilter = {
|
|
|
138
163
|
* only when the agent carries the creating subject as its viewer subject.
|
|
139
164
|
*/
|
|
140
165
|
agentOnly?: boolean | undefined;
|
|
166
|
+
/** Exact attempt whose grant snapshot must be revalidated in the content query. */
|
|
167
|
+
authorizedPersonalAttempt?:
|
|
168
|
+
| {
|
|
169
|
+
accountId: string;
|
|
170
|
+
workspaceId: string;
|
|
171
|
+
sessionId: string;
|
|
172
|
+
attemptId: string;
|
|
173
|
+
}
|
|
174
|
+
| undefined;
|
|
141
175
|
};
|
|
142
176
|
|
|
143
177
|
export type DocumentAuthority = {
|
|
@@ -146,6 +180,44 @@ export type DocumentAuthority = {
|
|
|
146
180
|
subjectId: string | null;
|
|
147
181
|
};
|
|
148
182
|
|
|
183
|
+
/** JSON-safe projection of the opaque capability minted by the API access layer. */
|
|
184
|
+
export type DocumentAccountAdminAuthorization = Readonly<{
|
|
185
|
+
authorizationId: string;
|
|
186
|
+
accountId: string;
|
|
187
|
+
actorSubjectId: string;
|
|
188
|
+
permission: "account:admin";
|
|
189
|
+
}>;
|
|
190
|
+
|
|
191
|
+
export type ReclassifyDocumentAuthorityInput = ReclassifyDocumentAuthorityRequest & {
|
|
192
|
+
accountId: string;
|
|
193
|
+
workspaceId: string;
|
|
194
|
+
documentId: string;
|
|
195
|
+
actorSubjectId: string;
|
|
196
|
+
accountAdminAuthorization: DocumentAccountAdminAuthorization | null;
|
|
197
|
+
};
|
|
198
|
+
|
|
199
|
+
export type RunDocumentDefaultCollectionBackfillInput =
|
|
200
|
+
RunDocumentDefaultCollectionBackfillRequest & {
|
|
201
|
+
accountId: string;
|
|
202
|
+
workspaceId: string;
|
|
203
|
+
actorSubjectId: string;
|
|
204
|
+
accountAdminAuthorization: DocumentAccountAdminAuthorization;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
export type DocumentMigrationAuditInput = {
|
|
208
|
+
accountId: string;
|
|
209
|
+
workspaceId: string;
|
|
210
|
+
actorSubjectId: string;
|
|
211
|
+
accountAdminAuthorization: DocumentAccountAdminAuthorization;
|
|
212
|
+
limit?: number | undefined;
|
|
213
|
+
cursor?: string | undefined;
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
export type AgentDocumentAuthorityContext = {
|
|
217
|
+
sessionId: string;
|
|
218
|
+
attemptId: string;
|
|
219
|
+
};
|
|
220
|
+
|
|
149
221
|
export type DocumentInventoryStatusCounts = Record<DocumentStatus, number>;
|
|
150
222
|
export type DocumentInventorySourceKindCounts = Record<KnowledgeSourceKind, number>;
|
|
151
223
|
export type DocumentInventoryAuthorityKindCounts = Record<DocumentAuthorityKind, number>;
|
|
@@ -196,6 +268,7 @@ export type EffectiveDocumentSearchInput = Omit<DocumentSearchInput, "access"> &
|
|
|
196
268
|
initiatingSubjectId: string;
|
|
197
269
|
/** Agent retrieval additionally enforces documents.agent_access. */
|
|
198
270
|
surface: "human" | "agent";
|
|
271
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
199
272
|
};
|
|
200
273
|
|
|
201
274
|
export type ListEffectiveIndexedDocumentsInput = {
|
|
@@ -205,6 +278,7 @@ export type ListEffectiveIndexedDocumentsInput = {
|
|
|
205
278
|
initiatingSubjectId: string;
|
|
206
279
|
checkpoint?: string | undefined;
|
|
207
280
|
limit?: number | undefined;
|
|
281
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
208
282
|
};
|
|
209
283
|
|
|
210
284
|
export type EffectiveKnowledgeBrowseInput = {
|
|
@@ -212,12 +286,15 @@ export type EffectiveKnowledgeBrowseInput = {
|
|
|
212
286
|
workspaceId: string;
|
|
213
287
|
/** Immutable human subject accepted for the logical request/turn. */
|
|
214
288
|
initiatingSubjectId: string;
|
|
289
|
+
/** Human inspection bypasses agent_access but still uses exact subject/RLS authority. */
|
|
290
|
+
surface?: "human" | "agent" | undefined;
|
|
215
291
|
/** Omit to browse top-level documents; pass a document record id for chunks. */
|
|
216
292
|
parentId?: string | undefined;
|
|
217
293
|
topic?: string | undefined;
|
|
218
294
|
sourceKinds?: KnowledgeSourceKind[] | undefined;
|
|
219
295
|
cursor?: string | undefined;
|
|
220
296
|
limit?: number | undefined;
|
|
297
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
221
298
|
};
|
|
222
299
|
|
|
223
300
|
export type DocumentIndexHooks = {
|
|
@@ -931,6 +1008,438 @@ export async function listDocumentBasesEnsuringDefault(
|
|
|
931
1008
|
return await listDocumentBases(db, input.workspaceId);
|
|
932
1009
|
}
|
|
933
1010
|
|
|
1011
|
+
export async function runDocumentDefaultCollectionBackfill(
|
|
1012
|
+
db: Database,
|
|
1013
|
+
input: RunDocumentDefaultCollectionBackfillInput,
|
|
1014
|
+
) {
|
|
1015
|
+
return await withDocumentAccountRls(
|
|
1016
|
+
db,
|
|
1017
|
+
input.accountId,
|
|
1018
|
+
input.workspaceId,
|
|
1019
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1020
|
+
async (scopedDb) => {
|
|
1021
|
+
const command = {
|
|
1022
|
+
accountId: input.accountId,
|
|
1023
|
+
workspaceId: input.workspaceId,
|
|
1024
|
+
actorSubjectId: input.actorSubjectId,
|
|
1025
|
+
runId: input.runId,
|
|
1026
|
+
operationId: input.operationId,
|
|
1027
|
+
batchSize: input.batchSize,
|
|
1028
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1029
|
+
};
|
|
1030
|
+
const rows = (await scopedDb.execute(sql`
|
|
1031
|
+
SELECT run_document_default_collection_backfill(
|
|
1032
|
+
${JSON.stringify(command)}::jsonb
|
|
1033
|
+
) AS result
|
|
1034
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1035
|
+
const row = rows[0];
|
|
1036
|
+
if (!row) throw new Error("Document Default collection backfill returned no result");
|
|
1037
|
+
return DocumentDefaultCollectionBackfill.parse(row.result);
|
|
1038
|
+
},
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
export async function listDocumentDefaultCollectionBackfillRuns(
|
|
1043
|
+
db: Database,
|
|
1044
|
+
input: DocumentMigrationAuditInput,
|
|
1045
|
+
) {
|
|
1046
|
+
const limit = validateDocumentMigrationAuditLimit(input.limit);
|
|
1047
|
+
const cursor = input.cursor
|
|
1048
|
+
? decodeDocumentMigrationAuditCursor(input.cursor, ["backfill-runs", input.accountId], true)
|
|
1049
|
+
: null;
|
|
1050
|
+
return await withDocumentAccountRls(
|
|
1051
|
+
db,
|
|
1052
|
+
input.accountId,
|
|
1053
|
+
input.workspaceId,
|
|
1054
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1055
|
+
async (scopedDb) => {
|
|
1056
|
+
const command = {
|
|
1057
|
+
accountId: input.accountId,
|
|
1058
|
+
workspaceId: input.workspaceId,
|
|
1059
|
+
actorSubjectId: input.actorSubjectId,
|
|
1060
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1061
|
+
limit: limit + 1,
|
|
1062
|
+
beforeStartedAt: cursor?.timestamp ?? null,
|
|
1063
|
+
beforeRunId: cursor?.id ?? null,
|
|
1064
|
+
};
|
|
1065
|
+
const rows = (await scopedDb.execute(sql`
|
|
1066
|
+
SELECT list_document_default_collection_backfill_runs(
|
|
1067
|
+
${JSON.stringify(command)}::jsonb
|
|
1068
|
+
) AS result
|
|
1069
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1070
|
+
const parsed = rows.map((row) => DocumentDefaultCollectionBackfillRunAudit.parse(row.result));
|
|
1071
|
+
const hasMore = parsed.length > limit;
|
|
1072
|
+
const runs = parsed.slice(0, limit);
|
|
1073
|
+
const tail = hasMore ? runs.at(-1) : null;
|
|
1074
|
+
return ListDocumentDefaultCollectionBackfillRunsResponse.parse({
|
|
1075
|
+
runs,
|
|
1076
|
+
hasMore,
|
|
1077
|
+
nextCursor: tail
|
|
1078
|
+
? encodeDocumentMigrationAuditCursor(["backfill-runs", input.accountId], {
|
|
1079
|
+
timestamp: tail.startedAt,
|
|
1080
|
+
id: tail.runId,
|
|
1081
|
+
})
|
|
1082
|
+
: null,
|
|
1083
|
+
});
|
|
1084
|
+
},
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
export async function getDocumentDefaultCollectionBackfillAudit(
|
|
1089
|
+
db: Database,
|
|
1090
|
+
input: DocumentMigrationAuditInput & {
|
|
1091
|
+
runId: string;
|
|
1092
|
+
operationCursor?: string | undefined;
|
|
1093
|
+
receiptCursor?: string | undefined;
|
|
1094
|
+
},
|
|
1095
|
+
) {
|
|
1096
|
+
const limit = validateDocumentMigrationAuditLimit(input.limit);
|
|
1097
|
+
const operationCursor = input.operationCursor
|
|
1098
|
+
? decodeDocumentMigrationAuditCursor(
|
|
1099
|
+
input.operationCursor,
|
|
1100
|
+
["backfill-operations", input.accountId, input.runId],
|
|
1101
|
+
true,
|
|
1102
|
+
)
|
|
1103
|
+
: null;
|
|
1104
|
+
const receiptCursor = input.receiptCursor
|
|
1105
|
+
? decodeDocumentMigrationAuditCursor(
|
|
1106
|
+
input.receiptCursor,
|
|
1107
|
+
["backfill-receipts", input.accountId, input.runId],
|
|
1108
|
+
false,
|
|
1109
|
+
)
|
|
1110
|
+
: null;
|
|
1111
|
+
return await withDocumentAccountRls(
|
|
1112
|
+
db,
|
|
1113
|
+
input.accountId,
|
|
1114
|
+
input.workspaceId,
|
|
1115
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1116
|
+
async (scopedDb) => {
|
|
1117
|
+
const command = {
|
|
1118
|
+
accountId: input.accountId,
|
|
1119
|
+
workspaceId: input.workspaceId,
|
|
1120
|
+
actorSubjectId: input.actorSubjectId,
|
|
1121
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1122
|
+
runId: input.runId,
|
|
1123
|
+
limit,
|
|
1124
|
+
operationBeforeCreatedAt: operationCursor?.timestamp ?? null,
|
|
1125
|
+
operationBeforeId: operationCursor?.id ?? null,
|
|
1126
|
+
receiptAfterWorkspaceId: receiptCursor?.id ?? null,
|
|
1127
|
+
};
|
|
1128
|
+
const rows = (await scopedDb.execute(sql`
|
|
1129
|
+
SELECT get_document_default_collection_backfill_audit(
|
|
1130
|
+
${JSON.stringify(command)}::jsonb
|
|
1131
|
+
) AS result
|
|
1132
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1133
|
+
const raw = rows[0]?.result;
|
|
1134
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
1135
|
+
throw new Error("document Default collection backfill audit returned no result");
|
|
1136
|
+
}
|
|
1137
|
+
const value = raw as Record<string, unknown>;
|
|
1138
|
+
const run = DocumentDefaultCollectionBackfillRunAudit.parse(value.run);
|
|
1139
|
+
const operations = Array.isArray(value.operations)
|
|
1140
|
+
? value.operations.map((operation) =>
|
|
1141
|
+
DocumentDefaultCollectionBackfillOperationAudit.parse(operation),
|
|
1142
|
+
)
|
|
1143
|
+
: [];
|
|
1144
|
+
const receipts = Array.isArray(value.receipts)
|
|
1145
|
+
? value.receipts.map((receipt) =>
|
|
1146
|
+
DocumentDefaultCollectionBackfillReceiptAudit.parse(receipt),
|
|
1147
|
+
)
|
|
1148
|
+
: [];
|
|
1149
|
+
const operationsHasMore = operations.length > limit;
|
|
1150
|
+
const receiptsHasMore = receipts.length > limit;
|
|
1151
|
+
const operationPage = operations.slice(0, limit);
|
|
1152
|
+
const receiptPage = receipts.slice(0, limit);
|
|
1153
|
+
const operationTail = operationsHasMore ? operationPage.at(-1) : null;
|
|
1154
|
+
const receiptTail = receiptsHasMore ? receiptPage.at(-1) : null;
|
|
1155
|
+
return DocumentDefaultCollectionBackfillAudit.parse({
|
|
1156
|
+
run,
|
|
1157
|
+
operations: operationPage,
|
|
1158
|
+
receipts: receiptPage,
|
|
1159
|
+
operationsHasMore,
|
|
1160
|
+
operationsNextCursor: operationTail
|
|
1161
|
+
? encodeDocumentMigrationAuditCursor(
|
|
1162
|
+
["backfill-operations", input.accountId, input.runId],
|
|
1163
|
+
{ timestamp: operationTail.createdAt, id: operationTail.operationId },
|
|
1164
|
+
)
|
|
1165
|
+
: null,
|
|
1166
|
+
receiptsHasMore,
|
|
1167
|
+
receiptsNextCursor: receiptTail
|
|
1168
|
+
? encodeDocumentMigrationAuditCursor(
|
|
1169
|
+
["backfill-receipts", input.accountId, input.runId],
|
|
1170
|
+
{ timestamp: null, id: receiptTail.workspaceId },
|
|
1171
|
+
)
|
|
1172
|
+
: null,
|
|
1173
|
+
});
|
|
1174
|
+
},
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
export async function listOrganizationDocumentAuthorityReclassifications(
|
|
1179
|
+
db: Database,
|
|
1180
|
+
input: DocumentMigrationAuditInput,
|
|
1181
|
+
) {
|
|
1182
|
+
const limit = validateDocumentMigrationAuditLimit(input.limit);
|
|
1183
|
+
const cursor = input.cursor
|
|
1184
|
+
? decodeDocumentMigrationAuditCursor(
|
|
1185
|
+
input.cursor,
|
|
1186
|
+
["organization-reclassifications", input.accountId],
|
|
1187
|
+
true,
|
|
1188
|
+
)
|
|
1189
|
+
: null;
|
|
1190
|
+
return await withDocumentAccountRls(
|
|
1191
|
+
db,
|
|
1192
|
+
input.accountId,
|
|
1193
|
+
input.workspaceId,
|
|
1194
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1195
|
+
async (scopedDb) => {
|
|
1196
|
+
const command = {
|
|
1197
|
+
accountId: input.accountId,
|
|
1198
|
+
workspaceId: input.workspaceId,
|
|
1199
|
+
actorSubjectId: input.actorSubjectId,
|
|
1200
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1201
|
+
limit: limit + 1,
|
|
1202
|
+
beforeCreatedAt: cursor?.timestamp ?? null,
|
|
1203
|
+
beforeOperationId: cursor?.id ?? null,
|
|
1204
|
+
};
|
|
1205
|
+
const rows = (await scopedDb.execute(sql`
|
|
1206
|
+
SELECT list_organization_document_authority_reclassifications(
|
|
1207
|
+
${JSON.stringify(command)}::jsonb
|
|
1208
|
+
) AS result
|
|
1209
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1210
|
+
const parsed = rows.map((row) =>
|
|
1211
|
+
OrganizationDocumentAuthorityReclassification.parse(row.result),
|
|
1212
|
+
);
|
|
1213
|
+
const hasMore = parsed.length > limit;
|
|
1214
|
+
const receipts = parsed.slice(0, limit);
|
|
1215
|
+
const tail = hasMore ? receipts.at(-1) : null;
|
|
1216
|
+
return ListOrganizationDocumentAuthorityReclassificationsResponse.parse({
|
|
1217
|
+
receipts,
|
|
1218
|
+
hasMore,
|
|
1219
|
+
nextCursor: tail
|
|
1220
|
+
? encodeDocumentMigrationAuditCursor(
|
|
1221
|
+
["organization-reclassifications", input.accountId],
|
|
1222
|
+
{ timestamp: tail.createdAt, id: tail.operationId },
|
|
1223
|
+
)
|
|
1224
|
+
: null,
|
|
1225
|
+
});
|
|
1226
|
+
},
|
|
1227
|
+
);
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
function validateDocumentMigrationAuditLimit(value: number | undefined): number {
|
|
1231
|
+
const limit = value ?? 50;
|
|
1232
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
|
|
1233
|
+
throw new Error("document migration audit limit must be between 1 and 100");
|
|
1234
|
+
}
|
|
1235
|
+
return limit;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
type DocumentMigrationAuditCursor = { timestamp: string | null; id: string };
|
|
1239
|
+
|
|
1240
|
+
function documentMigrationAuditCursorScope(parts: string[]): string {
|
|
1241
|
+
return createHash("sha256")
|
|
1242
|
+
.update(JSON.stringify(["document-migration-audit", 1, ...parts]), "utf8")
|
|
1243
|
+
.digest("hex")
|
|
1244
|
+
.slice(0, 32);
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
function encodeDocumentMigrationAuditCursor(
|
|
1248
|
+
parts: string[],
|
|
1249
|
+
cursor: DocumentMigrationAuditCursor,
|
|
1250
|
+
): string {
|
|
1251
|
+
return Buffer.from(
|
|
1252
|
+
JSON.stringify({
|
|
1253
|
+
v: 1,
|
|
1254
|
+
s: documentMigrationAuditCursorScope(parts),
|
|
1255
|
+
t: cursor.timestamp,
|
|
1256
|
+
i: cursor.id,
|
|
1257
|
+
}),
|
|
1258
|
+
"utf8",
|
|
1259
|
+
).toString("base64url");
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
function decodeDocumentMigrationAuditCursor(
|
|
1263
|
+
value: string,
|
|
1264
|
+
parts: string[],
|
|
1265
|
+
requireTimestamp: boolean,
|
|
1266
|
+
): DocumentMigrationAuditCursor {
|
|
1267
|
+
try {
|
|
1268
|
+
if (!value || value.length > 1_024) throw new Error("cursor length");
|
|
1269
|
+
const bytes = Buffer.from(value, "base64url");
|
|
1270
|
+
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
1271
|
+
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
1272
|
+
if (
|
|
1273
|
+
Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
|
|
1274
|
+
parsed.v !== 1 ||
|
|
1275
|
+
parsed.s !== documentMigrationAuditCursorScope(parts) ||
|
|
1276
|
+
typeof parsed.i !== "string" ||
|
|
1277
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(
|
|
1278
|
+
parsed.i,
|
|
1279
|
+
) ||
|
|
1280
|
+
(requireTimestamp
|
|
1281
|
+
? typeof parsed.t !== "string" || !Number.isFinite(new Date(parsed.t).getTime())
|
|
1282
|
+
: parsed.t !== null)
|
|
1283
|
+
) {
|
|
1284
|
+
throw new Error("cursor payload");
|
|
1285
|
+
}
|
|
1286
|
+
return { timestamp: typeof parsed.t === "string" ? parsed.t : null, id: parsed.i };
|
|
1287
|
+
} catch (error) {
|
|
1288
|
+
throw new Error("invalid document migration audit cursor", { cause: error });
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
export async function reclassifyDocumentAuthority(
|
|
1293
|
+
db: Database,
|
|
1294
|
+
input: ReclassifyDocumentAuthorityInput,
|
|
1295
|
+
) {
|
|
1296
|
+
return await withDocumentAccountRls(
|
|
1297
|
+
db,
|
|
1298
|
+
input.accountId,
|
|
1299
|
+
input.workspaceId,
|
|
1300
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1301
|
+
async (scopedDb) => {
|
|
1302
|
+
const command = {
|
|
1303
|
+
accountId: input.accountId,
|
|
1304
|
+
workspaceId: input.workspaceId,
|
|
1305
|
+
documentId: input.documentId,
|
|
1306
|
+
operationId: input.operationId,
|
|
1307
|
+
actorSubjectId: input.actorSubjectId,
|
|
1308
|
+
expectedAuthority: input.expectedAuthority,
|
|
1309
|
+
targetAuthorityKind: input.targetAuthorityKind,
|
|
1310
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1311
|
+
};
|
|
1312
|
+
const rows = (await scopedDb.execute(sql`
|
|
1313
|
+
SELECT reclassify_document_authority(${JSON.stringify(command)}::jsonb) AS result
|
|
1314
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1315
|
+
const row = rows[0];
|
|
1316
|
+
if (!row) throw new Error("Document authority reclassification returned no result");
|
|
1317
|
+
return DocumentAuthorityReclassification.parse(row.result);
|
|
1318
|
+
},
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
export async function listDocumentAuthorityReclassifications(
|
|
1323
|
+
db: Database,
|
|
1324
|
+
input: {
|
|
1325
|
+
accountId: string;
|
|
1326
|
+
workspaceId: string;
|
|
1327
|
+
documentId: string;
|
|
1328
|
+
actorSubjectId: string;
|
|
1329
|
+
limit?: number | undefined;
|
|
1330
|
+
cursor?: string | undefined;
|
|
1331
|
+
},
|
|
1332
|
+
) {
|
|
1333
|
+
const limit = input.limit ?? 50;
|
|
1334
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
|
|
1335
|
+
throw new Error("document authority receipt limit must be between 1 and 100");
|
|
1336
|
+
}
|
|
1337
|
+
const cursor = input.cursor
|
|
1338
|
+
? decodeDocumentAuthorityReclassificationCursor(input.cursor, input)
|
|
1339
|
+
: null;
|
|
1340
|
+
return await withDocumentAccountRls(
|
|
1341
|
+
db,
|
|
1342
|
+
input.accountId,
|
|
1343
|
+
input.workspaceId,
|
|
1344
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1345
|
+
async (scopedDb) => {
|
|
1346
|
+
const rows = (await scopedDb.execute(sql`
|
|
1347
|
+
SELECT list_document_authority_reclassifications(
|
|
1348
|
+
${input.accountId}::uuid,
|
|
1349
|
+
${input.workspaceId}::uuid,
|
|
1350
|
+
${input.actorSubjectId},
|
|
1351
|
+
${input.documentId}::uuid,
|
|
1352
|
+
${limit + 1}::integer,
|
|
1353
|
+
${cursor?.createdAt ?? null}::timestamptz,
|
|
1354
|
+
${cursor?.operationId ?? null}::uuid
|
|
1355
|
+
) AS result
|
|
1356
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1357
|
+
const parsed = rows.map((row) => DocumentAuthorityReclassification.parse(row.result));
|
|
1358
|
+
const hasMore = parsed.length > limit;
|
|
1359
|
+
const receipts = parsed.slice(0, limit);
|
|
1360
|
+
const tail = hasMore ? receipts.at(-1) : null;
|
|
1361
|
+
return ListDocumentAuthorityReclassificationsResponse.parse({
|
|
1362
|
+
receipts,
|
|
1363
|
+
hasMore,
|
|
1364
|
+
nextCursor: tail
|
|
1365
|
+
? encodeDocumentAuthorityReclassificationCursor(input, {
|
|
1366
|
+
createdAt: tail.createdAt,
|
|
1367
|
+
operationId: tail.operationId,
|
|
1368
|
+
})
|
|
1369
|
+
: null,
|
|
1370
|
+
});
|
|
1371
|
+
},
|
|
1372
|
+
);
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
type DocumentAuthorityReclassificationCursor = {
|
|
1376
|
+
createdAt: string;
|
|
1377
|
+
operationId: string;
|
|
1378
|
+
};
|
|
1379
|
+
|
|
1380
|
+
function documentAuthorityReclassificationCursorScope(input: {
|
|
1381
|
+
accountId: string;
|
|
1382
|
+
workspaceId: string;
|
|
1383
|
+
documentId: string;
|
|
1384
|
+
actorSubjectId: string;
|
|
1385
|
+
}): string {
|
|
1386
|
+
return createHash("sha256")
|
|
1387
|
+
.update(
|
|
1388
|
+
JSON.stringify([
|
|
1389
|
+
"document_authority_reclassification_cursor",
|
|
1390
|
+
1,
|
|
1391
|
+
input.accountId,
|
|
1392
|
+
input.workspaceId,
|
|
1393
|
+
input.documentId,
|
|
1394
|
+
input.actorSubjectId,
|
|
1395
|
+
]),
|
|
1396
|
+
"utf8",
|
|
1397
|
+
)
|
|
1398
|
+
.digest("hex")
|
|
1399
|
+
.slice(0, 32);
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
function encodeDocumentAuthorityReclassificationCursor(
|
|
1403
|
+
scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
|
|
1404
|
+
cursor: DocumentAuthorityReclassificationCursor,
|
|
1405
|
+
): string {
|
|
1406
|
+
return Buffer.from(
|
|
1407
|
+
JSON.stringify({
|
|
1408
|
+
v: 1,
|
|
1409
|
+
s: documentAuthorityReclassificationCursorScope(scope),
|
|
1410
|
+
t: cursor.createdAt,
|
|
1411
|
+
i: cursor.operationId,
|
|
1412
|
+
}),
|
|
1413
|
+
"utf8",
|
|
1414
|
+
).toString("base64url");
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
function decodeDocumentAuthorityReclassificationCursor(
|
|
1418
|
+
value: string,
|
|
1419
|
+
scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
|
|
1420
|
+
): DocumentAuthorityReclassificationCursor {
|
|
1421
|
+
try {
|
|
1422
|
+
if (!value || value.length > 1_024) throw new Error("cursor length");
|
|
1423
|
+
const bytes = Buffer.from(value, "base64url");
|
|
1424
|
+
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
1425
|
+
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
1426
|
+
if (
|
|
1427
|
+
Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
|
|
1428
|
+
parsed.v !== 1 ||
|
|
1429
|
+
parsed.s !== documentAuthorityReclassificationCursorScope(scope) ||
|
|
1430
|
+
typeof parsed.t !== "string" ||
|
|
1431
|
+
!Number.isFinite(new Date(parsed.t).getTime()) ||
|
|
1432
|
+
typeof parsed.i !== "string" ||
|
|
1433
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(parsed.i)
|
|
1434
|
+
) {
|
|
1435
|
+
throw new Error("cursor payload");
|
|
1436
|
+
}
|
|
1437
|
+
return { createdAt: parsed.t, operationId: parsed.i };
|
|
1438
|
+
} catch (error) {
|
|
1439
|
+
throw new Error("invalid document authority receipt cursor", { cause: error });
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
|
|
934
1443
|
export async function addDocumentToBase(
|
|
935
1444
|
db: Database,
|
|
936
1445
|
input: AddDocumentRequest & {
|
|
@@ -1042,36 +1551,53 @@ export async function addDocumentToBase(
|
|
|
1042
1551
|
.returning();
|
|
1043
1552
|
return mapDocument(updated ?? existing);
|
|
1044
1553
|
}
|
|
1045
|
-
const
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1554
|
+
const documentId = randomUUID();
|
|
1555
|
+
const row = await scopedDb.transaction(async (tx) => {
|
|
1556
|
+
const userAuthority =
|
|
1557
|
+
authority.kind === "personal"
|
|
1558
|
+
? await createPersonalDocumentAuthority(tx, {
|
|
1559
|
+
accountId: input.accountId,
|
|
1560
|
+
workspaceId: input.workspaceId,
|
|
1561
|
+
subjectId: authority.subjectId!,
|
|
1562
|
+
documentId,
|
|
1563
|
+
})
|
|
1564
|
+
: null;
|
|
1565
|
+
const [inserted] = await tx
|
|
1566
|
+
.insert(schema.documents)
|
|
1567
|
+
.values({
|
|
1568
|
+
id: documentId,
|
|
1569
|
+
accountId: input.accountId,
|
|
1570
|
+
workspaceId: input.workspaceId,
|
|
1571
|
+
baseId: input.baseId,
|
|
1572
|
+
fileId: input.fileId,
|
|
1573
|
+
status: "queued",
|
|
1574
|
+
title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
|
|
1575
|
+
parser: DEFAULT_DOCUMENT_PARSER,
|
|
1576
|
+
sourceKind: input.sourceKind ?? "manual_upload",
|
|
1577
|
+
sourceUri: cleanString(input.sourceUri) ?? null,
|
|
1578
|
+
sourceExternalId: cleanString(input.sourceExternalId) ?? null,
|
|
1579
|
+
sourceTitle: cleanString(input.sourceTitle) ?? null,
|
|
1580
|
+
sourceAuthor: cleanString(input.sourceAuthor) ?? null,
|
|
1581
|
+
sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
|
|
1582
|
+
sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
|
|
1583
|
+
sourceVersion: cleanString(input.sourceVersion) ?? null,
|
|
1584
|
+
knowledgeSourceIdentity,
|
|
1585
|
+
aclTags: cleanStringArray(input.aclTags),
|
|
1586
|
+
authorityKind: authority.kind,
|
|
1587
|
+
authorityWorkspaceId: userAuthority ? null : authority.workspaceId,
|
|
1588
|
+
authoritySubjectId: authority.subjectId,
|
|
1589
|
+
authorityId: userAuthority?.authorityId ?? null,
|
|
1590
|
+
ownerOrganizationMembershipId: userAuthority?.ownerOrganizationMembershipId ?? null,
|
|
1591
|
+
originWorkspaceId: input.workspaceId,
|
|
1592
|
+
visibility: authority.kind === "personal" ? "private" : "workspace",
|
|
1593
|
+
agentAccess: input.agentAccess ?? true,
|
|
1594
|
+
createdBy: fileAuthoritySubjectId,
|
|
1595
|
+
curationStatus: input.curationStatus ?? "none",
|
|
1596
|
+
updatedAt: now,
|
|
1597
|
+
})
|
|
1598
|
+
.returning();
|
|
1599
|
+
return inserted;
|
|
1600
|
+
});
|
|
1075
1601
|
if (!row) throw new Error("Failed to create document");
|
|
1076
1602
|
return mapDocument(row);
|
|
1077
1603
|
},
|
|
@@ -1105,7 +1631,7 @@ export async function moveDocumentToBase(
|
|
|
1105
1631
|
.from(schema.documents)
|
|
1106
1632
|
.where(
|
|
1107
1633
|
and(
|
|
1108
|
-
eq(schema.documents.
|
|
1634
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1109
1635
|
eq(schema.documents.id, input.documentId),
|
|
1110
1636
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1111
1637
|
),
|
|
@@ -1120,14 +1646,18 @@ export async function moveDocumentToBase(
|
|
|
1120
1646
|
throw new Error("document has no suggested base; pass targetBaseId");
|
|
1121
1647
|
}
|
|
1122
1648
|
if (targetBaseId === row.baseId) return mapDocument(row);
|
|
1123
|
-
|
|
1649
|
+
// A portable personal Document retains its immutable ingestion workspace
|
|
1650
|
+
// as provenance and physical storage. Management may be authorized from
|
|
1651
|
+
// another same-organization workspace, but filing still targets a base
|
|
1652
|
+
// in that origin workspace rather than silently copying authority/data.
|
|
1653
|
+
const base = await getDocumentBase(scopedDb, row.workspaceId, targetBaseId);
|
|
1124
1654
|
if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
|
|
1125
1655
|
const [conflict] = await scopedDb
|
|
1126
1656
|
.select({ id: schema.documents.id })
|
|
1127
1657
|
.from(schema.documents)
|
|
1128
1658
|
.where(
|
|
1129
1659
|
and(
|
|
1130
|
-
eq(schema.documents.workspaceId,
|
|
1660
|
+
eq(schema.documents.workspaceId, row.workspaceId),
|
|
1131
1661
|
eq(schema.documents.baseId, targetBaseId),
|
|
1132
1662
|
...(row.knowledgeSourceIdentity
|
|
1133
1663
|
? [eq(schema.documents.knowledgeSourceIdentity, row.knowledgeSourceIdentity)]
|
|
@@ -1151,7 +1681,7 @@ export async function moveDocumentToBase(
|
|
|
1151
1681
|
})
|
|
1152
1682
|
.where(
|
|
1153
1683
|
and(
|
|
1154
|
-
eq(schema.documents.workspaceId,
|
|
1684
|
+
eq(schema.documents.workspaceId, row.workspaceId),
|
|
1155
1685
|
eq(schema.documents.id, input.documentId),
|
|
1156
1686
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1157
1687
|
),
|
|
@@ -1163,7 +1693,7 @@ export async function moveDocumentToBase(
|
|
|
1163
1693
|
.set({ baseId: targetBaseId })
|
|
1164
1694
|
.where(
|
|
1165
1695
|
and(
|
|
1166
|
-
eq(schema.documentChunks.workspaceId,
|
|
1696
|
+
eq(schema.documentChunks.workspaceId, row.workspaceId),
|
|
1167
1697
|
eq(schema.documentChunks.documentId, input.documentId),
|
|
1168
1698
|
),
|
|
1169
1699
|
);
|
|
@@ -1198,7 +1728,7 @@ export async function deleteDocumentFromBase(
|
|
|
1198
1728
|
.from(schema.documents)
|
|
1199
1729
|
.where(
|
|
1200
1730
|
and(
|
|
1201
|
-
eq(schema.documents.
|
|
1731
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1202
1732
|
eq(schema.documents.id, input.documentId),
|
|
1203
1733
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1204
1734
|
),
|
|
@@ -1218,7 +1748,7 @@ export async function deleteDocumentFromBase(
|
|
|
1218
1748
|
.delete(schema.documents)
|
|
1219
1749
|
.where(
|
|
1220
1750
|
and(
|
|
1221
|
-
eq(schema.documents.
|
|
1751
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1222
1752
|
eq(schema.documents.id, input.documentId),
|
|
1223
1753
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1224
1754
|
),
|
|
@@ -1249,6 +1779,28 @@ export async function listDocuments(
|
|
|
1249
1779
|
});
|
|
1250
1780
|
}
|
|
1251
1781
|
|
|
1782
|
+
/**
|
|
1783
|
+
* List Documents the human can discover and manage from the requested
|
|
1784
|
+
* workspace. Organization Documents are account-wide, workspace Documents are
|
|
1785
|
+
* local, activated organization-user Documents are portable across the owner's
|
|
1786
|
+
* same-organization workspaces, and legacy personal rows remain anchored to
|
|
1787
|
+
* their ingestion workspace through documentAccessConditions.
|
|
1788
|
+
*/
|
|
1789
|
+
export async function listAccessibleDocuments(
|
|
1790
|
+
db: Database,
|
|
1791
|
+
workspaceId: string,
|
|
1792
|
+
access: DocumentAccessFilter,
|
|
1793
|
+
): Promise<Document[]> {
|
|
1794
|
+
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
1795
|
+
const rows = await scopedDb
|
|
1796
|
+
.select()
|
|
1797
|
+
.from(schema.documents)
|
|
1798
|
+
.where(and(...documentAccessConditions(workspaceId, access)))
|
|
1799
|
+
.orderBy(desc(schema.documents.updatedAt), asc(schema.documents.createdAt));
|
|
1800
|
+
return rows.map(mapDocument);
|
|
1801
|
+
});
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1252
1804
|
/**
|
|
1253
1805
|
* List newly ready documents in the same effective scope used by agent
|
|
1254
1806
|
* retrieval. The opaque checkpoint is bound to the account, requesting
|
|
@@ -1271,10 +1823,13 @@ export async function listEffectiveIndexedDocuments(
|
|
|
1271
1823
|
initiatingSubjectId,
|
|
1272
1824
|
})
|
|
1273
1825
|
: 0n;
|
|
1274
|
-
const access
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1826
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
1827
|
+
accountId: input.accountId,
|
|
1828
|
+
workspaceId: input.workspaceId,
|
|
1829
|
+
initiatingSubjectId,
|
|
1830
|
+
surface: "agent",
|
|
1831
|
+
agentAuthority: input.agentAuthority,
|
|
1832
|
+
});
|
|
1278
1833
|
const rows = await withDocumentAccountRls(
|
|
1279
1834
|
db,
|
|
1280
1835
|
input.accountId,
|
|
@@ -1405,17 +1960,37 @@ export async function getDocument(
|
|
|
1405
1960
|
.select()
|
|
1406
1961
|
.from(schema.documents)
|
|
1407
1962
|
.where(
|
|
1408
|
-
and(
|
|
1409
|
-
eq(schema.documents.workspaceId, workspaceId),
|
|
1410
|
-
eq(schema.documents.id, documentId),
|
|
1411
|
-
...documentAccessConditions(workspaceId, access),
|
|
1412
|
-
),
|
|
1963
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1413
1964
|
)
|
|
1414
1965
|
.limit(1);
|
|
1415
1966
|
return row ? mapDocument(row) : null;
|
|
1416
1967
|
});
|
|
1417
1968
|
}
|
|
1418
1969
|
|
|
1970
|
+
/**
|
|
1971
|
+
* Resolve the immutable source file through Document authority in the requested
|
|
1972
|
+
* workspace. The file remains physically owned by its ingestion workspace;
|
|
1973
|
+
* callers never gain generic access to that workspace's file inventory.
|
|
1974
|
+
*/
|
|
1975
|
+
export async function getDocumentOriginalFile(
|
|
1976
|
+
db: Database,
|
|
1977
|
+
input: {
|
|
1978
|
+
accountId: string;
|
|
1979
|
+
workspaceId: string;
|
|
1980
|
+
documentId: string;
|
|
1981
|
+
access: DocumentAccessFilter;
|
|
1982
|
+
},
|
|
1983
|
+
): Promise<FileAsset | null> {
|
|
1984
|
+
const subjectId = cleanString(input.access.viewerSubjectId ?? null);
|
|
1985
|
+
if (!subjectId || input.access.agentOnly) return null;
|
|
1986
|
+
return await resolveDocumentOriginalFileForSubject(db, {
|
|
1987
|
+
accountId: input.accountId,
|
|
1988
|
+
workspaceId: input.workspaceId,
|
|
1989
|
+
subjectId,
|
|
1990
|
+
documentId: input.documentId,
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1419
1994
|
/**
|
|
1420
1995
|
* Internal ingestion-only read used after the worker has independently resolved
|
|
1421
1996
|
* and fenced the immutable document authority tuple. It deliberately does not
|
|
@@ -1449,14 +2024,10 @@ export async function queueDocumentForReindex(
|
|
|
1449
2024
|
): Promise<Document> {
|
|
1450
2025
|
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
1451
2026
|
const [document] = await scopedDb
|
|
1452
|
-
.select(
|
|
2027
|
+
.select()
|
|
1453
2028
|
.from(schema.documents)
|
|
1454
2029
|
.where(
|
|
1455
|
-
and(
|
|
1456
|
-
eq(schema.documents.workspaceId, workspaceId),
|
|
1457
|
-
eq(schema.documents.id, documentId),
|
|
1458
|
-
...documentAccessConditions(workspaceId, access),
|
|
1459
|
-
),
|
|
2030
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1460
2031
|
)
|
|
1461
2032
|
.limit(1);
|
|
1462
2033
|
if (!document) throw new Error(`Document not found: ${documentId}`);
|
|
@@ -1469,11 +2040,7 @@ export async function queueDocumentForReindex(
|
|
|
1469
2040
|
updatedAt: new Date(),
|
|
1470
2041
|
})
|
|
1471
2042
|
.where(
|
|
1472
|
-
and(
|
|
1473
|
-
eq(schema.documents.workspaceId, workspaceId),
|
|
1474
|
-
eq(schema.documents.id, documentId),
|
|
1475
|
-
...documentAccessConditions(workspaceId, access),
|
|
1476
|
-
),
|
|
2043
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1477
2044
|
)
|
|
1478
2045
|
.returning();
|
|
1479
2046
|
if (!row) throw new Error(`Document not found: ${documentId}`);
|
|
@@ -1534,7 +2101,11 @@ export async function indexDocumentNow(
|
|
|
1534
2101
|
);
|
|
1535
2102
|
});
|
|
1536
2103
|
try {
|
|
1537
|
-
const
|
|
2104
|
+
const object = await retryWhileMissing(async () =>
|
|
2105
|
+
objectStorage.getObjectBytes(file.objectKey),
|
|
2106
|
+
);
|
|
2107
|
+
if (!object) throw new Error("document source object is missing");
|
|
2108
|
+
const bytes = object.bytes;
|
|
1538
2109
|
const parsed = await services.parser.parse(bytes, file);
|
|
1539
2110
|
// Knowledge drops (curationStatus 'pending') are curated between parse and
|
|
1540
2111
|
// chunking when a provider is enabled, so chunk metadata and base placement
|
|
@@ -1790,6 +2361,20 @@ export async function searchDocuments(
|
|
|
1790
2361
|
input: DocumentSearchInput,
|
|
1791
2362
|
services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
|
|
1792
2363
|
): Promise<DocumentSearchResult[]> {
|
|
2364
|
+
return (await searchDocumentCandidates(db, input, services)).results;
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2367
|
+
export type DocumentCandidateRelevanceFloor = {
|
|
2368
|
+
vectorScore: number;
|
|
2369
|
+
keywordScore: number;
|
|
2370
|
+
};
|
|
2371
|
+
|
|
2372
|
+
async function searchDocumentCandidates(
|
|
2373
|
+
db: Database,
|
|
2374
|
+
input: DocumentSearchInput,
|
|
2375
|
+
services: Pick<DocumentServices, "embedder">,
|
|
2376
|
+
relevanceFloor?: DocumentCandidateRelevanceFloor,
|
|
2377
|
+
): Promise<{ results: DocumentSearchResult[]; belowRelevanceFloor: number }> {
|
|
1793
2378
|
await assertDocumentAccountWorkspace(db, input.accountId, input.workspaceId);
|
|
1794
2379
|
const mode = input.mode ?? "hybrid";
|
|
1795
2380
|
const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
|
|
@@ -1814,7 +2399,28 @@ export async function searchDocuments(
|
|
|
1814
2399
|
if (mode === "keyword" || mode === "hybrid") {
|
|
1815
2400
|
rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
|
|
1816
2401
|
}
|
|
1817
|
-
|
|
2402
|
+
const merged = mergeDocumentSearchRows(rows, mode);
|
|
2403
|
+
return selectDocumentSearchCandidateWindow(merged, limit, relevanceFloor);
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
export function selectDocumentSearchCandidateWindow(
|
|
2407
|
+
merged: DocumentSearchResult[],
|
|
2408
|
+
limit: number,
|
|
2409
|
+
relevanceFloor?: DocumentCandidateRelevanceFloor,
|
|
2410
|
+
): { results: DocumentSearchResult[]; belowRelevanceFloor: number } {
|
|
2411
|
+
const boundedLimit = Math.min(Math.max(Math.trunc(limit), 1), 50);
|
|
2412
|
+
if (!relevanceFloor) {
|
|
2413
|
+
return { results: merged.slice(0, boundedLimit), belowRelevanceFloor: 0 };
|
|
2414
|
+
}
|
|
2415
|
+
let belowRelevanceFloor = 0;
|
|
2416
|
+
const relevant = merged.filter((result) => {
|
|
2417
|
+
const included =
|
|
2418
|
+
(result.vectorScore !== null && result.vectorScore >= relevanceFloor.vectorScore) ||
|
|
2419
|
+
(result.keywordScore !== null && result.keywordScore >= relevanceFloor.keywordScore);
|
|
2420
|
+
if (!included) belowRelevanceFloor += 1;
|
|
2421
|
+
return included;
|
|
2422
|
+
});
|
|
2423
|
+
return { results: relevant.slice(0, boundedLimit), belowRelevanceFloor };
|
|
1818
2424
|
}
|
|
1819
2425
|
|
|
1820
2426
|
/**
|
|
@@ -1831,6 +2437,13 @@ export async function searchEffectiveDocuments(
|
|
|
1831
2437
|
if (!initiatingSubjectId) {
|
|
1832
2438
|
throw new Error("effective document retrieval requires an initiating subject");
|
|
1833
2439
|
}
|
|
2440
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2441
|
+
accountId: input.accountId,
|
|
2442
|
+
workspaceId: input.workspaceId,
|
|
2443
|
+
initiatingSubjectId,
|
|
2444
|
+
surface: input.surface,
|
|
2445
|
+
agentAuthority: input.agentAuthority,
|
|
2446
|
+
});
|
|
1834
2447
|
return await searchDocuments(
|
|
1835
2448
|
db,
|
|
1836
2449
|
{
|
|
@@ -1844,10 +2457,7 @@ export async function searchEffectiveDocuments(
|
|
|
1844
2457
|
aclTags: input.aclTags,
|
|
1845
2458
|
// Construct the lower-level access filter here instead of spreading the
|
|
1846
2459
|
// caller input, so an untyped/legacy access override is always ignored.
|
|
1847
|
-
access
|
|
1848
|
-
viewerSubjectId: initiatingSubjectId,
|
|
1849
|
-
...(input.surface === "agent" ? { agentOnly: true } : {}),
|
|
1850
|
-
},
|
|
2460
|
+
access,
|
|
1851
2461
|
},
|
|
1852
2462
|
services,
|
|
1853
2463
|
);
|
|
@@ -1865,13 +2475,45 @@ export async function searchEffectiveKnowledge(
|
|
|
1865
2475
|
services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
|
|
1866
2476
|
): Promise<KnowledgeSearchResponse> {
|
|
1867
2477
|
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1868
|
-
const
|
|
2478
|
+
const requestedLimit = Math.min(Math.max(input.limit ?? 5, 1), KNOWLEDGE_SEARCH_MAX_RESULTS);
|
|
2479
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2480
|
+
accountId: input.accountId,
|
|
2481
|
+
workspaceId: input.workspaceId,
|
|
2482
|
+
initiatingSubjectId,
|
|
2483
|
+
surface: input.surface,
|
|
2484
|
+
agentAuthority: input.agentAuthority,
|
|
2485
|
+
});
|
|
2486
|
+
// Pull a bounded surplus so the permission-safe result set can still satisfy
|
|
2487
|
+
// the caller after relevance filtering and exact-content deduplication.
|
|
2488
|
+
const candidateLimit = Math.min(requestedLimit * 4, KNOWLEDGE_SEARCH_MAX_RESULTS);
|
|
2489
|
+
const rankedSelection = await searchDocumentCandidates(
|
|
1869
2490
|
db,
|
|
1870
|
-
{
|
|
2491
|
+
{
|
|
2492
|
+
accountId: input.accountId,
|
|
2493
|
+
workspaceId: input.workspaceId,
|
|
2494
|
+
query: input.query,
|
|
2495
|
+
limit: candidateLimit,
|
|
2496
|
+
...(input.baseIds ? { baseIds: input.baseIds } : {}),
|
|
2497
|
+
...(input.mode ? { mode: input.mode } : {}),
|
|
2498
|
+
...(input.sourceKinds ? { sourceKinds: input.sourceKinds } : {}),
|
|
2499
|
+
...(input.aclTags ? { aclTags: input.aclTags } : {}),
|
|
2500
|
+
access,
|
|
2501
|
+
},
|
|
1871
2502
|
services,
|
|
2503
|
+
{
|
|
2504
|
+
vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
|
|
2505
|
+
keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
|
|
2506
|
+
},
|
|
1872
2507
|
);
|
|
1873
|
-
|
|
1874
|
-
|
|
2508
|
+
const ranked = rankedSelection.results;
|
|
2509
|
+
if (ranked.length === 0) {
|
|
2510
|
+
return selectKnowledgeSearchResults({
|
|
2511
|
+
candidates: [],
|
|
2512
|
+
rankedCandidateCount: 0,
|
|
2513
|
+
requestedLimit,
|
|
2514
|
+
alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
|
|
2515
|
+
});
|
|
2516
|
+
}
|
|
1875
2517
|
const current = await withDocumentAccountRls(
|
|
1876
2518
|
db,
|
|
1877
2519
|
input.accountId,
|
|
@@ -1883,6 +2525,8 @@ export async function searchEffectiveKnowledge(
|
|
|
1883
2525
|
chunk: schema.documentChunks,
|
|
1884
2526
|
document: schema.documents,
|
|
1885
2527
|
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2528
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
2529
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
1886
2530
|
})
|
|
1887
2531
|
.from(schema.documentChunks)
|
|
1888
2532
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
@@ -1900,23 +2544,399 @@ export async function searchEffectiveKnowledge(
|
|
|
1900
2544
|
),
|
|
1901
2545
|
);
|
|
1902
2546
|
const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
|
|
1903
|
-
return {
|
|
1904
|
-
|
|
2547
|
+
return selectKnowledgeSearchResults({
|
|
2548
|
+
rankedCandidateCount: ranked.length,
|
|
2549
|
+
requestedLimit,
|
|
2550
|
+
alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
|
|
2551
|
+
candidates: ranked.flatMap((rankedResult) => {
|
|
1905
2552
|
const row = currentByChunkId.get(rankedResult.chunkId);
|
|
1906
2553
|
if (!row) return [];
|
|
1907
2554
|
return [
|
|
1908
2555
|
{
|
|
1909
|
-
record: knowledgeChunkRecord(row.document, row.chunk, row.citation
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
2556
|
+
record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
2557
|
+
previousChunkId: row.previousChunkId,
|
|
2558
|
+
nextChunkId: row.nextChunkId,
|
|
2559
|
+
}),
|
|
2560
|
+
semanticScore: rankedResult.score,
|
|
2561
|
+
matchType: rankedResult.matchType,
|
|
2562
|
+
vectorScore: rankedResult.vectorScore,
|
|
2563
|
+
keywordScore: rankedResult.keywordScore,
|
|
1916
2564
|
},
|
|
1917
2565
|
];
|
|
1918
2566
|
}),
|
|
2567
|
+
});
|
|
2568
|
+
}
|
|
2569
|
+
|
|
2570
|
+
type KnowledgeSearchCandidate = {
|
|
2571
|
+
record: KnowledgeRecord;
|
|
2572
|
+
semanticScore: number;
|
|
2573
|
+
matchType: DocumentSearchMode;
|
|
2574
|
+
vectorScore: number | null;
|
|
2575
|
+
keywordScore: number | null;
|
|
2576
|
+
};
|
|
2577
|
+
|
|
2578
|
+
type KnowledgeSearchSelectionInput = {
|
|
2579
|
+
candidates: KnowledgeSearchCandidate[];
|
|
2580
|
+
rankedCandidateCount: number;
|
|
2581
|
+
requestedLimit: number;
|
|
2582
|
+
alreadyBelowRelevanceFloor?: number | undefined;
|
|
2583
|
+
now?: Date | undefined;
|
|
2584
|
+
};
|
|
2585
|
+
|
|
2586
|
+
/**
|
|
2587
|
+
* Deterministic, content-safe final selection over already-authorized and
|
|
2588
|
+
* freshly rechecked Knowledge candidates. Exported for boundary tests; callers
|
|
2589
|
+
* must never use it as a substitute for the database authorization pass above.
|
|
2590
|
+
*/
|
|
2591
|
+
export function selectKnowledgeSearchResults(
|
|
2592
|
+
input: KnowledgeSearchSelectionInput,
|
|
2593
|
+
): KnowledgeSearchResponse {
|
|
2594
|
+
const requestedLimit = Math.min(
|
|
2595
|
+
Math.max(Math.trunc(input.requestedLimit), 1),
|
|
2596
|
+
KNOWLEDGE_SEARCH_MAX_RESULTS,
|
|
2597
|
+
);
|
|
2598
|
+
const nowMs = (input.now ?? new Date()).getTime();
|
|
2599
|
+
let belowRelevanceFloor = Math.min(
|
|
2600
|
+
Math.max(0, Math.trunc(input.alreadyBelowRelevanceFloor ?? 0)),
|
|
2601
|
+
KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
|
|
2602
|
+
);
|
|
2603
|
+
const relevant: KnowledgeSearchResult[] = [];
|
|
2604
|
+
for (const candidate of input.candidates) {
|
|
2605
|
+
const relevanceSignals: Array<"vector" | "keyword"> = [];
|
|
2606
|
+
if (
|
|
2607
|
+
candidate.vectorScore !== null &&
|
|
2608
|
+
candidate.vectorScore >= KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE
|
|
2609
|
+
) {
|
|
2610
|
+
relevanceSignals.push("vector");
|
|
2611
|
+
}
|
|
2612
|
+
if (
|
|
2613
|
+
candidate.keywordScore !== null &&
|
|
2614
|
+
candidate.keywordScore >= KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE
|
|
2615
|
+
) {
|
|
2616
|
+
relevanceSignals.push("keyword");
|
|
2617
|
+
}
|
|
2618
|
+
if (relevanceSignals.length === 0) {
|
|
2619
|
+
belowRelevanceFloor = Math.min(belowRelevanceFloor + 1, KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS);
|
|
2620
|
+
continue;
|
|
2621
|
+
}
|
|
2622
|
+
const freshness = knowledgeFreshness(candidate.record.quality.freshnessAt, nowMs);
|
|
2623
|
+
const qualityAdjustment = freshness === "current" ? 0.02 : freshness === "aging" ? 0.01 : 0;
|
|
2624
|
+
relevant.push({
|
|
2625
|
+
record: candidate.record,
|
|
2626
|
+
retrieval: {
|
|
2627
|
+
score: roundScore(Math.min(1, candidate.semanticScore + qualityAdjustment)),
|
|
2628
|
+
semanticScore: roundScore(candidate.semanticScore),
|
|
2629
|
+
matchType: candidate.matchType,
|
|
2630
|
+
vectorScore: candidate.vectorScore === null ? null : roundScore(candidate.vectorScore),
|
|
2631
|
+
keywordScore: candidate.keywordScore === null ? null : roundScore(candidate.keywordScore),
|
|
2632
|
+
relevanceSignals,
|
|
2633
|
+
freshness,
|
|
2634
|
+
qualityAdjustment,
|
|
2635
|
+
duplicateCount: 0,
|
|
2636
|
+
},
|
|
2637
|
+
});
|
|
2638
|
+
}
|
|
2639
|
+
relevant.sort(compareKnowledgeSearchResults);
|
|
2640
|
+
|
|
2641
|
+
const deduped: KnowledgeSearchResult[] = [];
|
|
2642
|
+
const byContent = new Map<string, number>();
|
|
2643
|
+
let asDuplicate = 0;
|
|
2644
|
+
for (const result of relevant) {
|
|
2645
|
+
const key = knowledgeTextualContentKey(result.record);
|
|
2646
|
+
const retainedIndex = byContent.get(key);
|
|
2647
|
+
if (retainedIndex === undefined) {
|
|
2648
|
+
byContent.set(key, deduped.length);
|
|
2649
|
+
deduped.push(result);
|
|
2650
|
+
continue;
|
|
2651
|
+
}
|
|
2652
|
+
asDuplicate += 1;
|
|
2653
|
+
const retained = deduped[retainedIndex]!;
|
|
2654
|
+
retained.retrieval.duplicateCount += 1;
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
const forLimit = Math.max(0, deduped.length - requestedLimit);
|
|
2658
|
+
const bounded = deduped.slice(0, requestedLimit);
|
|
2659
|
+
let forResponseBudget = 0;
|
|
2660
|
+
let response = knowledgeSearchResponse({
|
|
2661
|
+
results: bounded,
|
|
2662
|
+
rankedCandidateCount: input.rankedCandidateCount,
|
|
2663
|
+
recheckedCandidateCount: input.candidates.length,
|
|
2664
|
+
belowRelevanceFloor,
|
|
2665
|
+
asDuplicate,
|
|
2666
|
+
forLimit,
|
|
2667
|
+
forResponseBudget,
|
|
2668
|
+
});
|
|
2669
|
+
while (knowledgeResponseBytes(response) > KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES) {
|
|
2670
|
+
if (bounded.length === 0) {
|
|
2671
|
+
throw new Error("knowledge search selection facts exceed the response budget");
|
|
2672
|
+
}
|
|
2673
|
+
bounded.pop();
|
|
2674
|
+
forResponseBudget += 1;
|
|
2675
|
+
response = knowledgeSearchResponse({
|
|
2676
|
+
results: bounded,
|
|
2677
|
+
rankedCandidateCount: input.rankedCandidateCount,
|
|
2678
|
+
recheckedCandidateCount: input.candidates.length,
|
|
2679
|
+
belowRelevanceFloor,
|
|
2680
|
+
asDuplicate,
|
|
2681
|
+
forLimit,
|
|
2682
|
+
forResponseBudget,
|
|
2683
|
+
});
|
|
2684
|
+
}
|
|
2685
|
+
return response;
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
function compareKnowledgeSearchResults(
|
|
2689
|
+
left: KnowledgeSearchResult,
|
|
2690
|
+
right: KnowledgeSearchResult,
|
|
2691
|
+
): number {
|
|
2692
|
+
return (
|
|
2693
|
+
right.retrieval.score - left.retrieval.score ||
|
|
2694
|
+
right.retrieval.semanticScore - left.retrieval.semanticScore ||
|
|
2695
|
+
(right.retrieval.vectorScore ?? 0) - (left.retrieval.vectorScore ?? 0) ||
|
|
2696
|
+
(right.retrieval.keywordScore ?? 0) - (left.retrieval.keywordScore ?? 0) ||
|
|
2697
|
+
(left.record.id === right.record.id ? 0 : left.record.id < right.record.id ? -1 : 1)
|
|
2698
|
+
);
|
|
2699
|
+
}
|
|
2700
|
+
|
|
2701
|
+
function knowledgeFreshness(value: string, nowMs: number): "current" | "aging" | "stale" {
|
|
2702
|
+
const freshnessMs = Date.parse(value);
|
|
2703
|
+
if (!Number.isFinite(freshnessMs)) return "stale";
|
|
2704
|
+
const ageDays = Math.max(0, (nowMs - freshnessMs) / 86_400_000);
|
|
2705
|
+
if (ageDays <= 90) return "current";
|
|
2706
|
+
return ageDays <= 365 ? "aging" : "stale";
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
function knowledgeTextualContentKey(record: KnowledgeRecord): string {
|
|
2710
|
+
return createHash("sha256")
|
|
2711
|
+
.update("opengeni:knowledge-search-content:v1\0")
|
|
2712
|
+
.update(record.title)
|
|
2713
|
+
.update("\0")
|
|
2714
|
+
.update(
|
|
2715
|
+
JSON.stringify({
|
|
2716
|
+
body: record.content.body,
|
|
2717
|
+
summary: record.content.summary,
|
|
2718
|
+
topics: record.content.topics,
|
|
2719
|
+
}),
|
|
2720
|
+
)
|
|
2721
|
+
.digest("hex");
|
|
2722
|
+
}
|
|
2723
|
+
|
|
2724
|
+
function knowledgeSearchResponse(input: {
|
|
2725
|
+
results: KnowledgeSearchResult[];
|
|
2726
|
+
rankedCandidateCount: number;
|
|
2727
|
+
recheckedCandidateCount: number;
|
|
2728
|
+
belowRelevanceFloor: number;
|
|
2729
|
+
asDuplicate: number;
|
|
2730
|
+
forLimit: number;
|
|
2731
|
+
forResponseBudget: number;
|
|
2732
|
+
}): KnowledgeSearchResponse {
|
|
2733
|
+
const selection = {
|
|
2734
|
+
relevanceFloor: {
|
|
2735
|
+
policy: "any_signal" as const,
|
|
2736
|
+
vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE as 0.52,
|
|
2737
|
+
keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE as 0.01,
|
|
2738
|
+
},
|
|
2739
|
+
dedupe: { policy: "exact_textual_content" as const },
|
|
2740
|
+
candidates: {
|
|
2741
|
+
ranked: input.rankedCandidateCount,
|
|
2742
|
+
rechecked: input.recheckedCandidateCount,
|
|
2743
|
+
omittedOnRecheck: Math.max(0, input.rankedCandidateCount - input.recheckedCandidateCount),
|
|
2744
|
+
},
|
|
2745
|
+
omitted: {
|
|
2746
|
+
belowRelevanceFloor: input.belowRelevanceFloor,
|
|
2747
|
+
asDuplicate: input.asDuplicate,
|
|
2748
|
+
forLimit: input.forLimit,
|
|
2749
|
+
forResponseBudget: input.forResponseBudget,
|
|
2750
|
+
},
|
|
2751
|
+
budget: {
|
|
2752
|
+
maxResults: KNOWLEDGE_SEARCH_MAX_RESULTS as 50,
|
|
2753
|
+
maxResponseBytes: KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES as 65_536,
|
|
2754
|
+
responseBytes: 0,
|
|
2755
|
+
tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
|
|
2756
|
+
estimatedTokens: 0,
|
|
2757
|
+
maxEstimatedTokens: Math.ceil(
|
|
2758
|
+
KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2759
|
+
) as 16_384,
|
|
2760
|
+
},
|
|
2761
|
+
};
|
|
2762
|
+
const response: KnowledgeSearchResponse = {
|
|
2763
|
+
results: [...input.results],
|
|
2764
|
+
selection,
|
|
1919
2765
|
};
|
|
2766
|
+
// The counters themselves contribute a few bytes. Iterate to a fixed point so
|
|
2767
|
+
// responseBytes describes the actual serialized response, not an approximation.
|
|
2768
|
+
for (let index = 0; index < 8; index += 1) {
|
|
2769
|
+
const responseBytes = knowledgeResponseBytes(response);
|
|
2770
|
+
const estimatedTokens = Math.ceil(
|
|
2771
|
+
responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2772
|
+
);
|
|
2773
|
+
if (
|
|
2774
|
+
response.selection.budget.responseBytes === responseBytes &&
|
|
2775
|
+
response.selection.budget.estimatedTokens === estimatedTokens
|
|
2776
|
+
) {
|
|
2777
|
+
break;
|
|
2778
|
+
}
|
|
2779
|
+
response.selection.budget.responseBytes = responseBytes;
|
|
2780
|
+
response.selection.budget.estimatedTokens = estimatedTokens;
|
|
2781
|
+
}
|
|
2782
|
+
return response;
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2785
|
+
function knowledgeResponseBytes(response: KnowledgeSearchResponse): number {
|
|
2786
|
+
return Buffer.byteLength(JSON.stringify(response), "utf8");
|
|
2787
|
+
}
|
|
2788
|
+
|
|
2789
|
+
type KnowledgeBrowseEntry = {
|
|
2790
|
+
record: KnowledgeRecord;
|
|
2791
|
+
cursorAfter: string;
|
|
2792
|
+
};
|
|
2793
|
+
|
|
2794
|
+
/**
|
|
2795
|
+
* Bound a browse page without skipping records. Tail records omitted for the
|
|
2796
|
+
* response budget remain behind the returned cursor. If one complete record is
|
|
2797
|
+
* itself too large, return a deterministic discovery projection; knowledge_get
|
|
2798
|
+
* remains the freshly authorized full-record path.
|
|
2799
|
+
*/
|
|
2800
|
+
export function selectKnowledgeBrowseRecords(input: {
|
|
2801
|
+
entries: KnowledgeBrowseEntry[];
|
|
2802
|
+
hasMoreAfterEntries: boolean;
|
|
2803
|
+
}): KnowledgeBrowseResponse {
|
|
2804
|
+
const entries = input.entries.slice(0, KNOWLEDGE_BROWSE_MAX_LIMIT);
|
|
2805
|
+
const selected = entries.map((entry) => entry.record);
|
|
2806
|
+
let omittedForResponseBudget = 0;
|
|
2807
|
+
let compactedRecordCount = 0;
|
|
2808
|
+
let response = knowledgeBrowseResponse({
|
|
2809
|
+
records: selected,
|
|
2810
|
+
nextCursor: input.hasMoreAfterEntries ? (entries.at(-1)?.cursorAfter ?? null) : null,
|
|
2811
|
+
hasMore: input.hasMoreAfterEntries,
|
|
2812
|
+
omittedForResponseBudget,
|
|
2813
|
+
compactedRecordCount,
|
|
2814
|
+
});
|
|
2815
|
+
while (
|
|
2816
|
+
selected.length > 1 &&
|
|
2817
|
+
knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
|
|
2818
|
+
) {
|
|
2819
|
+
selected.pop();
|
|
2820
|
+
omittedForResponseBudget += 1;
|
|
2821
|
+
response = knowledgeBrowseResponse({
|
|
2822
|
+
records: selected,
|
|
2823
|
+
nextCursor: entries[selected.length - 1]?.cursorAfter ?? null,
|
|
2824
|
+
hasMore: true,
|
|
2825
|
+
omittedForResponseBudget,
|
|
2826
|
+
compactedRecordCount,
|
|
2827
|
+
});
|
|
2828
|
+
}
|
|
2829
|
+
if (
|
|
2830
|
+
selected.length === 1 &&
|
|
2831
|
+
knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
|
|
2832
|
+
) {
|
|
2833
|
+
selected[0] = compactKnowledgeBrowseRecord(selected[0]!);
|
|
2834
|
+
compactedRecordCount = 1;
|
|
2835
|
+
response = knowledgeBrowseResponse({
|
|
2836
|
+
records: selected,
|
|
2837
|
+
nextCursor:
|
|
2838
|
+
omittedForResponseBudget > 0 || input.hasMoreAfterEntries
|
|
2839
|
+
? (entries[0]?.cursorAfter ?? null)
|
|
2840
|
+
: null,
|
|
2841
|
+
hasMore: omittedForResponseBudget > 0 || input.hasMoreAfterEntries,
|
|
2842
|
+
omittedForResponseBudget,
|
|
2843
|
+
compactedRecordCount,
|
|
2844
|
+
});
|
|
2845
|
+
}
|
|
2846
|
+
if (knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES) {
|
|
2847
|
+
throw new Error("knowledge browse discovery projection exceeds the response budget");
|
|
2848
|
+
}
|
|
2849
|
+
return response;
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
function compactKnowledgeBrowseRecord(record: KnowledgeRecord): KnowledgeRecord {
|
|
2853
|
+
const fields = new Set<KnowledgeRecord["projection"]["fields"][number]>([
|
|
2854
|
+
...record.projection.fields,
|
|
2855
|
+
"content.body",
|
|
2856
|
+
"content.summary",
|
|
2857
|
+
"content.topics",
|
|
2858
|
+
"content.metadata",
|
|
2859
|
+
"provenance.source.uri",
|
|
2860
|
+
"provenance.source.externalId",
|
|
2861
|
+
"provenance.source.title",
|
|
2862
|
+
"provenance.source.author",
|
|
2863
|
+
"provenance.source.version",
|
|
2864
|
+
"provenance.citation",
|
|
2865
|
+
]);
|
|
2866
|
+
return {
|
|
2867
|
+
...record,
|
|
2868
|
+
content: {
|
|
2869
|
+
format: "markdown",
|
|
2870
|
+
body: null,
|
|
2871
|
+
summary: null,
|
|
2872
|
+
topics: [],
|
|
2873
|
+
metadata: {},
|
|
2874
|
+
},
|
|
2875
|
+
provenance: {
|
|
2876
|
+
...record.provenance,
|
|
2877
|
+
source: {
|
|
2878
|
+
...record.provenance.source,
|
|
2879
|
+
uri: null,
|
|
2880
|
+
externalId: null,
|
|
2881
|
+
title: null,
|
|
2882
|
+
author: null,
|
|
2883
|
+
version: null,
|
|
2884
|
+
},
|
|
2885
|
+
citation: null,
|
|
2886
|
+
},
|
|
2887
|
+
links: record.links.filter((link) => link.target.kind === "knowledge"),
|
|
2888
|
+
projection: {
|
|
2889
|
+
truncated: true,
|
|
2890
|
+
fields: [...fields].sort(),
|
|
2891
|
+
},
|
|
2892
|
+
};
|
|
2893
|
+
}
|
|
2894
|
+
|
|
2895
|
+
function knowledgeBrowseResponse(input: {
|
|
2896
|
+
records: KnowledgeRecord[];
|
|
2897
|
+
nextCursor: string | null;
|
|
2898
|
+
hasMore: boolean;
|
|
2899
|
+
omittedForResponseBudget: number;
|
|
2900
|
+
compactedRecordCount: number;
|
|
2901
|
+
}): KnowledgeBrowseResponse {
|
|
2902
|
+
const response: KnowledgeBrowseResponse = {
|
|
2903
|
+
records: [...input.records],
|
|
2904
|
+
nextCursor: input.nextCursor,
|
|
2905
|
+
hasMore: input.hasMore,
|
|
2906
|
+
selection: {
|
|
2907
|
+
omitted: { forResponseBudget: input.omittedForResponseBudget },
|
|
2908
|
+
compactedRecordCount: input.compactedRecordCount,
|
|
2909
|
+
budget: {
|
|
2910
|
+
maxResults: KNOWLEDGE_BROWSE_MAX_LIMIT as 50,
|
|
2911
|
+
maxResponseBytes: KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES as 65_536,
|
|
2912
|
+
responseBytes: 0,
|
|
2913
|
+
tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
|
|
2914
|
+
estimatedTokens: 0,
|
|
2915
|
+
maxEstimatedTokens: Math.ceil(
|
|
2916
|
+
KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2917
|
+
) as 16_384,
|
|
2918
|
+
},
|
|
2919
|
+
},
|
|
2920
|
+
};
|
|
2921
|
+
for (let index = 0; index < 8; index += 1) {
|
|
2922
|
+
const responseBytes = knowledgeBrowseResponseBytes(response);
|
|
2923
|
+
const estimatedTokens = Math.ceil(
|
|
2924
|
+
responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2925
|
+
);
|
|
2926
|
+
if (
|
|
2927
|
+
response.selection.budget.responseBytes === responseBytes &&
|
|
2928
|
+
response.selection.budget.estimatedTokens === estimatedTokens
|
|
2929
|
+
) {
|
|
2930
|
+
break;
|
|
2931
|
+
}
|
|
2932
|
+
response.selection.budget.responseBytes = responseBytes;
|
|
2933
|
+
response.selection.budget.estimatedTokens = estimatedTokens;
|
|
2934
|
+
}
|
|
2935
|
+
return response;
|
|
2936
|
+
}
|
|
2937
|
+
|
|
2938
|
+
function knowledgeBrowseResponseBytes(response: KnowledgeBrowseResponse): number {
|
|
2939
|
+
return Buffer.byteLength(JSON.stringify(response), "utf8");
|
|
1920
2940
|
}
|
|
1921
2941
|
|
|
1922
2942
|
/** Fetch one stable Knowledge record with a fresh authorization check. */
|
|
@@ -1926,12 +2946,20 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1926
2946
|
accountId: string;
|
|
1927
2947
|
workspaceId: string;
|
|
1928
2948
|
initiatingSubjectId: string;
|
|
2949
|
+
surface?: "human" | "agent" | undefined;
|
|
1929
2950
|
id: string;
|
|
2951
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
1930
2952
|
},
|
|
1931
2953
|
): Promise<KnowledgeRecord | null> {
|
|
1932
2954
|
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1933
2955
|
const target = parseKnowledgeRecordId(input.id);
|
|
1934
|
-
const access
|
|
2956
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2957
|
+
accountId: input.accountId,
|
|
2958
|
+
workspaceId: input.workspaceId,
|
|
2959
|
+
initiatingSubjectId,
|
|
2960
|
+
surface: input.surface ?? "agent",
|
|
2961
|
+
agentAuthority: input.agentAuthority,
|
|
2962
|
+
});
|
|
1935
2963
|
return await withDocumentAccountRls(
|
|
1936
2964
|
db,
|
|
1937
2965
|
input.accountId,
|
|
@@ -1943,8 +2971,17 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1943
2971
|
.select({
|
|
1944
2972
|
document: schema.documents,
|
|
1945
2973
|
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2974
|
+
firstChunkId: schema.documentChunks.id,
|
|
1946
2975
|
})
|
|
1947
2976
|
.from(schema.documents)
|
|
2977
|
+
.leftJoin(
|
|
2978
|
+
schema.documentChunks,
|
|
2979
|
+
and(
|
|
2980
|
+
eq(schema.documentChunks.accountId, schema.documents.accountId),
|
|
2981
|
+
eq(schema.documentChunks.documentId, schema.documents.id),
|
|
2982
|
+
eq(schema.documentChunks.chunkIndex, 0),
|
|
2983
|
+
),
|
|
2984
|
+
)
|
|
1948
2985
|
.where(
|
|
1949
2986
|
and(
|
|
1950
2987
|
eq(schema.documents.accountId, input.accountId),
|
|
@@ -1954,13 +2991,15 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1954
2991
|
),
|
|
1955
2992
|
)
|
|
1956
2993
|
.limit(1);
|
|
1957
|
-
return row ? knowledgeDocumentRecord(row.document, row.citation) : null;
|
|
2994
|
+
return row ? knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId) : null;
|
|
1958
2995
|
}
|
|
1959
2996
|
const [row] = await scopedDb
|
|
1960
2997
|
.select({
|
|
1961
2998
|
chunk: schema.documentChunks,
|
|
1962
2999
|
document: schema.documents,
|
|
1963
3000
|
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
3001
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
3002
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
1964
3003
|
})
|
|
1965
3004
|
.from(schema.documentChunks)
|
|
1966
3005
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
@@ -1974,7 +3013,12 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1974
3013
|
),
|
|
1975
3014
|
)
|
|
1976
3015
|
.limit(1);
|
|
1977
|
-
return row
|
|
3016
|
+
return row
|
|
3017
|
+
? knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
3018
|
+
previousChunkId: row.previousChunkId,
|
|
3019
|
+
nextChunkId: row.nextChunkId,
|
|
3020
|
+
})
|
|
3021
|
+
: null;
|
|
1978
3022
|
},
|
|
1979
3023
|
);
|
|
1980
3024
|
}
|
|
@@ -2005,7 +3049,7 @@ export async function browseEffectiveKnowledge(
|
|
|
2005
3049
|
if (parent && (topic || sourceKinds.length > 0)) {
|
|
2006
3050
|
throw new Error("knowledge browse document contents do not accept topic/source filters");
|
|
2007
3051
|
}
|
|
2008
|
-
const cursorScope = {
|
|
3052
|
+
const cursorScope: KnowledgeBrowseCursorScope = {
|
|
2009
3053
|
accountId: input.accountId,
|
|
2010
3054
|
workspaceId: input.workspaceId,
|
|
2011
3055
|
initiatingSubjectId,
|
|
@@ -2013,11 +3057,15 @@ export async function browseEffectiveKnowledge(
|
|
|
2013
3057
|
topic,
|
|
2014
3058
|
sourceKinds,
|
|
2015
3059
|
};
|
|
2016
|
-
const
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
3060
|
+
const topLevelAfter =
|
|
3061
|
+
!parent && input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
|
|
3062
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
3063
|
+
accountId: input.accountId,
|
|
3064
|
+
workspaceId: input.workspaceId,
|
|
3065
|
+
initiatingSubjectId,
|
|
3066
|
+
surface: input.surface ?? "agent",
|
|
3067
|
+
agentAuthority: input.agentAuthority,
|
|
3068
|
+
});
|
|
2021
3069
|
return await withDocumentAccountRls(
|
|
2022
3070
|
db,
|
|
2023
3071
|
input.accountId,
|
|
@@ -2026,7 +3074,7 @@ export async function browseEffectiveKnowledge(
|
|
|
2026
3074
|
async (scopedDb) => {
|
|
2027
3075
|
if (parent) {
|
|
2028
3076
|
const [authorizedParent] = await scopedDb
|
|
2029
|
-
.select({ id: schema.documents.id })
|
|
3077
|
+
.select({ id: schema.documents.id, indexSequence: schema.documents.indexSequence })
|
|
2030
3078
|
.from(schema.documents)
|
|
2031
3079
|
.where(
|
|
2032
3080
|
and(
|
|
@@ -2037,12 +3085,29 @@ export async function browseEffectiveKnowledge(
|
|
|
2037
3085
|
),
|
|
2038
3086
|
)
|
|
2039
3087
|
.limit(1);
|
|
2040
|
-
if (!authorizedParent)
|
|
3088
|
+
if (!authorizedParent) {
|
|
3089
|
+
return selectKnowledgeBrowseRecords({ entries: [], hasMoreAfterEntries: false });
|
|
3090
|
+
}
|
|
3091
|
+
if (authorizedParent.indexSequence === null) {
|
|
3092
|
+
throw new Error("ready knowledge document is missing its index revision");
|
|
3093
|
+
}
|
|
3094
|
+
const parentCursorScope: KnowledgeBrowseCursorScope = {
|
|
3095
|
+
...cursorScope,
|
|
3096
|
+
parentRevision: authorizedParent.indexSequence.toString(),
|
|
3097
|
+
};
|
|
3098
|
+
const after = input.cursor
|
|
3099
|
+
? decodeKnowledgeBrowseCursor(input.cursor, parentCursorScope)
|
|
3100
|
+
: 0n;
|
|
3101
|
+
if (after > 2_147_483_648n) {
|
|
3102
|
+
throw new Error("invalid knowledge browse cursor");
|
|
3103
|
+
}
|
|
2041
3104
|
const rows = await scopedDb
|
|
2042
3105
|
.select({
|
|
2043
3106
|
chunk: schema.documentChunks,
|
|
2044
3107
|
document: schema.documents,
|
|
2045
3108
|
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
3109
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
3110
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
2046
3111
|
})
|
|
2047
3112
|
.from(schema.documentChunks)
|
|
2048
3113
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
@@ -2052,6 +3117,7 @@ export async function browseEffectiveKnowledge(
|
|
|
2052
3117
|
eq(schema.documentChunks.documentId, parent.id),
|
|
2053
3118
|
gt(schema.documentChunks.chunkIndex, Number(after) - 1),
|
|
2054
3119
|
eq(schema.documents.status, "ready"),
|
|
3120
|
+
eq(schema.documents.indexSequence, authorizedParent.indexSequence),
|
|
2055
3121
|
...documentAccessConditions(input.workspaceId, access),
|
|
2056
3122
|
),
|
|
2057
3123
|
)
|
|
@@ -2059,22 +3125,26 @@ export async function browseEffectiveKnowledge(
|
|
|
2059
3125
|
.limit(limit + 1);
|
|
2060
3126
|
const hasMore = rows.length > limit;
|
|
2061
3127
|
const page = rows.slice(0, limit);
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
3128
|
+
return selectKnowledgeBrowseRecords({
|
|
3129
|
+
entries: page.map((row) => ({
|
|
3130
|
+
record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
3131
|
+
previousChunkId: row.previousChunkId,
|
|
3132
|
+
nextChunkId: row.nextChunkId,
|
|
3133
|
+
}),
|
|
3134
|
+
cursorAfter: encodeKnowledgeBrowseCursor(
|
|
3135
|
+
parentCursorScope,
|
|
3136
|
+
BigInt(row.chunk.chunkIndex + 1),
|
|
3137
|
+
),
|
|
3138
|
+
})),
|
|
3139
|
+
hasMoreAfterEntries: hasMore,
|
|
3140
|
+
});
|
|
2071
3141
|
}
|
|
2072
3142
|
|
|
2073
3143
|
const conditions: SQL[] = [
|
|
2074
3144
|
eq(schema.documents.accountId, input.accountId),
|
|
2075
3145
|
eq(schema.documents.status, "ready"),
|
|
2076
3146
|
isNotNull(schema.documents.indexSequence),
|
|
2077
|
-
gt(schema.documents.indexSequence,
|
|
3147
|
+
gt(schema.documents.indexSequence, topLevelAfter),
|
|
2078
3148
|
...documentAccessConditions(input.workspaceId, access),
|
|
2079
3149
|
];
|
|
2080
3150
|
if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
|
|
@@ -2084,22 +3154,34 @@ export async function browseEffectiveKnowledge(
|
|
|
2084
3154
|
.select({
|
|
2085
3155
|
document: schema.documents,
|
|
2086
3156
|
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
3157
|
+
firstChunkId: schema.documentChunks.id,
|
|
2087
3158
|
})
|
|
2088
3159
|
.from(schema.documents)
|
|
3160
|
+
.leftJoin(
|
|
3161
|
+
schema.documentChunks,
|
|
3162
|
+
and(
|
|
3163
|
+
eq(schema.documentChunks.accountId, schema.documents.accountId),
|
|
3164
|
+
eq(schema.documentChunks.documentId, schema.documents.id),
|
|
3165
|
+
eq(schema.documentChunks.chunkIndex, 0),
|
|
3166
|
+
),
|
|
3167
|
+
)
|
|
2089
3168
|
.where(and(...conditions))
|
|
2090
3169
|
.orderBy(asc(schema.documents.indexSequence))
|
|
2091
3170
|
.limit(limit + 1);
|
|
2092
3171
|
const hasMore = rows.length > limit;
|
|
2093
3172
|
const page = rows.slice(0, limit);
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
:
|
|
2101
|
-
|
|
2102
|
-
|
|
3173
|
+
return selectKnowledgeBrowseRecords({
|
|
3174
|
+
entries: page.map((row) => {
|
|
3175
|
+
if (row.document.indexSequence === null) {
|
|
3176
|
+
throw new Error("ready knowledge document is missing its index revision");
|
|
3177
|
+
}
|
|
3178
|
+
return {
|
|
3179
|
+
record: knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId),
|
|
3180
|
+
cursorAfter: encodeKnowledgeBrowseCursor(cursorScope, row.document.indexSequence),
|
|
3181
|
+
};
|
|
3182
|
+
}),
|
|
3183
|
+
hasMoreAfterEntries: hasMore,
|
|
3184
|
+
});
|
|
2103
3185
|
},
|
|
2104
3186
|
);
|
|
2105
3187
|
}
|
|
@@ -2109,6 +3191,7 @@ type KnowledgeBrowseCursorScope = {
|
|
|
2109
3191
|
workspaceId: string;
|
|
2110
3192
|
initiatingSubjectId: string;
|
|
2111
3193
|
parentId: string | null;
|
|
3194
|
+
parentRevision?: string | null | undefined;
|
|
2112
3195
|
topic: string | null;
|
|
2113
3196
|
sourceKinds: readonly string[];
|
|
2114
3197
|
};
|
|
@@ -2118,8 +3201,9 @@ export function encodeKnowledgeBrowseCursor(
|
|
|
2118
3201
|
position: bigint,
|
|
2119
3202
|
): string {
|
|
2120
3203
|
if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
|
|
3204
|
+
const version = knowledgeBrowseCursorVersion(scope);
|
|
2121
3205
|
return Buffer.from(
|
|
2122
|
-
JSON.stringify({ v:
|
|
3206
|
+
JSON.stringify({ v: version, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
|
|
2123
3207
|
"utf8",
|
|
2124
3208
|
).toString("base64url");
|
|
2125
3209
|
}
|
|
@@ -2135,9 +3219,10 @@ export function decodeKnowledgeBrowseCursor(
|
|
|
2135
3219
|
const bytes = Buffer.from(value, "base64url");
|
|
2136
3220
|
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
2137
3221
|
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
3222
|
+
const expectedVersion = knowledgeBrowseCursorVersion(scope);
|
|
2138
3223
|
if (
|
|
2139
3224
|
Object.keys(parsed).sort().join(",") !== "q,s,v" ||
|
|
2140
|
-
parsed.v !==
|
|
3225
|
+
parsed.v !== expectedVersion ||
|
|
2141
3226
|
typeof parsed.s !== "string" ||
|
|
2142
3227
|
typeof parsed.q !== "string" ||
|
|
2143
3228
|
!/^(0|[1-9][0-9]*)$/.test(parsed.q)
|
|
@@ -2162,8 +3247,9 @@ export function decodeKnowledgeBrowseCursor(
|
|
|
2162
3247
|
}
|
|
2163
3248
|
|
|
2164
3249
|
function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
|
|
2165
|
-
|
|
2166
|
-
|
|
3250
|
+
const version = knowledgeBrowseCursorVersion(scope);
|
|
3251
|
+
const hash = createHash("sha256")
|
|
3252
|
+
.update(`opengeni:knowledge-browse-cursor:v${version}\0`)
|
|
2167
3253
|
.update(scope.accountId)
|
|
2168
3254
|
.update("\0")
|
|
2169
3255
|
.update(scope.workspaceId)
|
|
@@ -2171,13 +3257,23 @@ function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
|
|
|
2171
3257
|
.update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
|
|
2172
3258
|
.update("\0")
|
|
2173
3259
|
.update(scope.parentId ?? "")
|
|
2174
|
-
.update("\0")
|
|
3260
|
+
.update("\0");
|
|
3261
|
+
if (version === 2) hash.update(scope.parentRevision!).update("\0");
|
|
3262
|
+
return hash
|
|
2175
3263
|
.update(scope.topic ?? "")
|
|
2176
3264
|
.update("\0")
|
|
2177
3265
|
.update([...scope.sourceKinds].sort().join("\0"))
|
|
2178
3266
|
.digest("hex");
|
|
2179
3267
|
}
|
|
2180
3268
|
|
|
3269
|
+
function knowledgeBrowseCursorVersion(scope: KnowledgeBrowseCursorScope): 1 | 2 {
|
|
3270
|
+
if (!scope.parentId) return 1;
|
|
3271
|
+
if (!scope.parentRevision || !/^[1-9][0-9]*$/.test(scope.parentRevision)) {
|
|
3272
|
+
throw new Error("knowledge browse parent cursor requires an exact document revision");
|
|
3273
|
+
}
|
|
3274
|
+
return 2;
|
|
3275
|
+
}
|
|
3276
|
+
|
|
2181
3277
|
function parseKnowledgeRecordId(value: string): {
|
|
2182
3278
|
kind: "document" | "document_chunk";
|
|
2183
3279
|
id: string;
|
|
@@ -2193,6 +3289,7 @@ function parseKnowledgeRecordId(value: string): {
|
|
|
2193
3289
|
function knowledgeDocumentRecord(
|
|
2194
3290
|
document: typeof schema.documents.$inferSelect,
|
|
2195
3291
|
citation: unknown = null,
|
|
3292
|
+
firstChunkId: string | null = null,
|
|
2196
3293
|
): KnowledgeRecord {
|
|
2197
3294
|
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2198
3295
|
const projected = projectKnowledgeRecord({
|
|
@@ -2216,7 +3313,20 @@ function knowledgeDocumentRecord(
|
|
|
2216
3313
|
},
|
|
2217
3314
|
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2218
3315
|
quality: knowledgeQuality(document),
|
|
2219
|
-
links:
|
|
3316
|
+
links: [
|
|
3317
|
+
...(firstChunkId
|
|
3318
|
+
? [
|
|
3319
|
+
{
|
|
3320
|
+
relation: "contents" as const,
|
|
3321
|
+
target: {
|
|
3322
|
+
kind: "knowledge" as const,
|
|
3323
|
+
id: `document_chunk:${firstChunkId}` as const,
|
|
3324
|
+
},
|
|
3325
|
+
},
|
|
3326
|
+
]
|
|
3327
|
+
: []),
|
|
3328
|
+
...knowledgeSourceLinks(projected.source.uri),
|
|
3329
|
+
],
|
|
2220
3330
|
projection: projected.projection,
|
|
2221
3331
|
};
|
|
2222
3332
|
}
|
|
@@ -2225,6 +3335,10 @@ function knowledgeChunkRecord(
|
|
|
2225
3335
|
document: typeof schema.documents.$inferSelect,
|
|
2226
3336
|
chunk: typeof schema.documentChunks.$inferSelect,
|
|
2227
3337
|
citation: unknown = null,
|
|
3338
|
+
traversal: {
|
|
3339
|
+
previousChunkId: string | null;
|
|
3340
|
+
nextChunkId: string | null;
|
|
3341
|
+
} = { previousChunkId: null, nextChunkId: null },
|
|
2228
3342
|
): KnowledgeRecord {
|
|
2229
3343
|
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2230
3344
|
const projected = projectKnowledgeRecord({
|
|
@@ -2249,7 +3363,32 @@ function knowledgeChunkRecord(
|
|
|
2249
3363
|
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2250
3364
|
quality: knowledgeQuality(document),
|
|
2251
3365
|
links: [
|
|
2252
|
-
{
|
|
3366
|
+
{
|
|
3367
|
+
relation: "parent",
|
|
3368
|
+
target: { kind: "knowledge", id: `document:${document.id}` },
|
|
3369
|
+
},
|
|
3370
|
+
...(traversal.previousChunkId
|
|
3371
|
+
? [
|
|
3372
|
+
{
|
|
3373
|
+
relation: "previous" as const,
|
|
3374
|
+
target: {
|
|
3375
|
+
kind: "knowledge" as const,
|
|
3376
|
+
id: `document_chunk:${traversal.previousChunkId}` as const,
|
|
3377
|
+
},
|
|
3378
|
+
},
|
|
3379
|
+
]
|
|
3380
|
+
: []),
|
|
3381
|
+
...(traversal.nextChunkId
|
|
3382
|
+
? [
|
|
3383
|
+
{
|
|
3384
|
+
relation: "next" as const,
|
|
3385
|
+
target: {
|
|
3386
|
+
kind: "knowledge" as const,
|
|
3387
|
+
id: `document_chunk:${traversal.nextChunkId}` as const,
|
|
3388
|
+
},
|
|
3389
|
+
},
|
|
3390
|
+
]
|
|
3391
|
+
: []),
|
|
2253
3392
|
...knowledgeSourceLinks(projected.source.uri),
|
|
2254
3393
|
],
|
|
2255
3394
|
projection: projected.projection,
|
|
@@ -2285,6 +3424,35 @@ function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"
|
|
|
2285
3424
|
return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
|
|
2286
3425
|
}
|
|
2287
3426
|
|
|
3427
|
+
/**
|
|
3428
|
+
* These structural targets are selected inside the same authorization-scoped
|
|
3429
|
+
* transaction as their owning record. Only opaque ids are projected; titles,
|
|
3430
|
+
* source fields, and content require a subsequent freshly authorized get.
|
|
3431
|
+
*/
|
|
3432
|
+
function knowledgePreviousChunkIdProjection(): SQL<string | null> {
|
|
3433
|
+
return sql<string | null>`(
|
|
3434
|
+
select knowledge_previous_chunk.id
|
|
3435
|
+
from document_chunks knowledge_previous_chunk
|
|
3436
|
+
where knowledge_previous_chunk.account_id = ${schema.documentChunks.accountId}
|
|
3437
|
+
and knowledge_previous_chunk.document_id = ${schema.documentChunks.documentId}
|
|
3438
|
+
and knowledge_previous_chunk.chunk_index < ${schema.documentChunks.chunkIndex}
|
|
3439
|
+
order by knowledge_previous_chunk.chunk_index desc
|
|
3440
|
+
limit 1
|
|
3441
|
+
)`;
|
|
3442
|
+
}
|
|
3443
|
+
|
|
3444
|
+
function knowledgeNextChunkIdProjection(): SQL<string | null> {
|
|
3445
|
+
return sql<string | null>`(
|
|
3446
|
+
select knowledge_next_chunk.id
|
|
3447
|
+
from document_chunks knowledge_next_chunk
|
|
3448
|
+
where knowledge_next_chunk.account_id = ${schema.documentChunks.accountId}
|
|
3449
|
+
and knowledge_next_chunk.document_id = ${schema.documentChunks.documentId}
|
|
3450
|
+
and knowledge_next_chunk.chunk_index > ${schema.documentChunks.chunkIndex}
|
|
3451
|
+
order by knowledge_next_chunk.chunk_index asc
|
|
3452
|
+
limit 1
|
|
3453
|
+
)`;
|
|
3454
|
+
}
|
|
3455
|
+
|
|
2288
3456
|
async function vectorSearchDocuments(
|
|
2289
3457
|
db: Database,
|
|
2290
3458
|
input: DocumentSearchInput,
|
|
@@ -2329,7 +3497,7 @@ async function vectorSearchDocuments(
|
|
|
2329
3497
|
.from(schema.documentChunks)
|
|
2330
3498
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
2331
3499
|
.where(and(...documentSearchConditions(input, services.embedder.model)))
|
|
2332
|
-
.orderBy(distance)
|
|
3500
|
+
.orderBy(distance, asc(schema.documentChunks.id))
|
|
2333
3501
|
.limit(limit),
|
|
2334
3502
|
);
|
|
2335
3503
|
return rows.map((row) => ({
|
|
@@ -2385,7 +3553,7 @@ async function keywordSearchDocuments(
|
|
|
2385
3553
|
sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
|
|
2386
3554
|
),
|
|
2387
3555
|
)
|
|
2388
|
-
.orderBy(desc(rank))
|
|
3556
|
+
.orderBy(desc(rank), asc(schema.documentChunks.id))
|
|
2389
3557
|
.limit(limit),
|
|
2390
3558
|
);
|
|
2391
3559
|
return rows.map((row) => ({
|
|
@@ -2535,6 +3703,33 @@ async function assertDocumentAccountWorkspace(
|
|
|
2535
3703
|
return context;
|
|
2536
3704
|
}
|
|
2537
3705
|
|
|
3706
|
+
export async function resolveEffectiveDocumentAccess(
|
|
3707
|
+
_db: Database,
|
|
3708
|
+
input: {
|
|
3709
|
+
accountId: string;
|
|
3710
|
+
workspaceId: string;
|
|
3711
|
+
initiatingSubjectId: string;
|
|
3712
|
+
surface: "human" | "agent";
|
|
3713
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
3714
|
+
},
|
|
3715
|
+
): Promise<DocumentAccessFilter> {
|
|
3716
|
+
if (input.surface === "human") {
|
|
3717
|
+
return { viewerSubjectId: input.initiatingSubjectId };
|
|
3718
|
+
}
|
|
3719
|
+
return {
|
|
3720
|
+
agentOnly: true,
|
|
3721
|
+
viewerSubjectId: input.initiatingSubjectId,
|
|
3722
|
+
authorizedPersonalAttempt: input.agentAuthority
|
|
3723
|
+
? {
|
|
3724
|
+
accountId: input.accountId,
|
|
3725
|
+
workspaceId: input.workspaceId,
|
|
3726
|
+
sessionId: input.agentAuthority.sessionId,
|
|
3727
|
+
attemptId: input.agentAuthority.attemptId,
|
|
3728
|
+
}
|
|
3729
|
+
: undefined,
|
|
3730
|
+
};
|
|
3731
|
+
}
|
|
3732
|
+
|
|
2538
3733
|
/**
|
|
2539
3734
|
* Visibility/agent scoping shared by every document read path. Fail-closed:
|
|
2540
3735
|
* with no filter supplied, private documents are invisible.
|
|
@@ -2549,11 +3744,33 @@ function documentAccessConditions(
|
|
|
2549
3744
|
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
2550
3745
|
);
|
|
2551
3746
|
const viewer = cleanString(access?.viewerSubjectId ?? null);
|
|
3747
|
+
const personalAttempt = access?.authorizedPersonalAttempt;
|
|
3748
|
+
const authorizedPersonal = personalAttempt
|
|
3749
|
+
? sql`${schema.documents.id} IN (
|
|
3750
|
+
SELECT resolve_session_attempt_personal_document_reads(
|
|
3751
|
+
${personalAttempt.accountId}::uuid,
|
|
3752
|
+
${personalAttempt.workspaceId}::uuid,
|
|
3753
|
+
${personalAttempt.sessionId}::uuid,
|
|
3754
|
+
${personalAttempt.attemptId}::uuid
|
|
3755
|
+
)
|
|
3756
|
+
)`
|
|
3757
|
+
: sql`false`;
|
|
2552
3758
|
const personal = viewer
|
|
2553
3759
|
? and(
|
|
2554
3760
|
eq(schema.documents.authorityKind, "personal"),
|
|
2555
|
-
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
2556
3761
|
eq(schema.documents.authoritySubjectId, viewer),
|
|
3762
|
+
access?.agentOnly
|
|
3763
|
+
? (or(
|
|
3764
|
+
and(
|
|
3765
|
+
isNull(schema.documents.authorityId),
|
|
3766
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
3767
|
+
),
|
|
3768
|
+
and(isNotNull(schema.documents.authorityId), authorizedPersonal),
|
|
3769
|
+
) ?? authorizedPersonal)
|
|
3770
|
+
: (or(
|
|
3771
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
3772
|
+
isNull(schema.documents.authorityWorkspaceId),
|
|
3773
|
+
) ?? eq(schema.documents.authorityWorkspaceId, workspaceId)),
|
|
2557
3774
|
)
|
|
2558
3775
|
: undefined;
|
|
2559
3776
|
const authority = viewer
|
|
@@ -2575,13 +3792,22 @@ function documentAccessConditions(
|
|
|
2575
3792
|
function documentMatchesAccess(
|
|
2576
3793
|
document: Pick<
|
|
2577
3794
|
DocumentAccessRecord,
|
|
2578
|
-
|
|
3795
|
+
| "id"
|
|
3796
|
+
| "authorityId"
|
|
3797
|
+
| "authorityKind"
|
|
3798
|
+
| "authorityWorkspaceId"
|
|
3799
|
+
| "authoritySubjectId"
|
|
3800
|
+
| "agentAccess"
|
|
2579
3801
|
>,
|
|
2580
3802
|
workspaceId: string,
|
|
2581
3803
|
access: DocumentAccessFilter | undefined,
|
|
2582
3804
|
): boolean {
|
|
2583
3805
|
if (access?.agentOnly) {
|
|
2584
|
-
return
|
|
3806
|
+
return (
|
|
3807
|
+
document.agentAccess &&
|
|
3808
|
+
(document.authorityKind !== "personal" || document.authorityId === null) &&
|
|
3809
|
+
canViewDocument(document, access.viewerSubjectId, workspaceId)
|
|
3810
|
+
);
|
|
2585
3811
|
}
|
|
2586
3812
|
return canViewDocument(document, access?.viewerSubjectId, workspaceId);
|
|
2587
3813
|
}
|
|
@@ -2599,9 +3825,9 @@ function canonicalDocumentAuthoritySubject(value: unknown): string | undefined {
|
|
|
2599
3825
|
/**
|
|
2600
3826
|
* Whether a single already-fetched document is readable in this workspace.
|
|
2601
3827
|
*
|
|
2602
|
-
* Keep this compatibility predicate as strict as SQL/RLS: personal
|
|
2603
|
-
* remains
|
|
2604
|
-
* authority
|
|
3828
|
+
* Keep this compatibility predicate as strict as SQL/RLS: legacy personal
|
|
3829
|
+
* authority remains origin-workspace anchored while activated personal
|
|
3830
|
+
* authority has a null workspace and follows the exact owner within the org.
|
|
2605
3831
|
*/
|
|
2606
3832
|
export function canViewDocument(
|
|
2607
3833
|
document: Pick<DocumentAccessRecord, "authorityKind" | "authoritySubjectId"> & {
|
|
@@ -2614,21 +3840,31 @@ export function canViewDocument(
|
|
|
2614
3840
|
return document.authorityWorkspaceId === null && document.authoritySubjectId === null;
|
|
2615
3841
|
}
|
|
2616
3842
|
const normalizedWorkspaceId = cleanString(workspaceId ?? null);
|
|
2617
|
-
if (!normalizedWorkspaceId
|
|
3843
|
+
if (!normalizedWorkspaceId) {
|
|
2618
3844
|
return false;
|
|
2619
3845
|
}
|
|
2620
3846
|
if (document.authorityKind === "workspace") {
|
|
2621
|
-
return
|
|
3847
|
+
return (
|
|
3848
|
+
document.authorityWorkspaceId === normalizedWorkspaceId &&
|
|
3849
|
+
document.authoritySubjectId === null
|
|
3850
|
+
);
|
|
2622
3851
|
}
|
|
2623
3852
|
if (document.authorityKind === "personal") {
|
|
2624
3853
|
const authoritySubjectId = canonicalDocumentAuthoritySubject(document.authoritySubjectId);
|
|
2625
3854
|
const viewer = canonicalDocumentAuthoritySubject(viewerSubjectId);
|
|
2626
|
-
return
|
|
3855
|
+
return (
|
|
3856
|
+
!!authoritySubjectId &&
|
|
3857
|
+
authoritySubjectId === viewer &&
|
|
3858
|
+
(document.authorityWorkspaceId === null ||
|
|
3859
|
+
document.authorityWorkspaceId === normalizedWorkspaceId)
|
|
3860
|
+
);
|
|
2627
3861
|
}
|
|
2628
3862
|
return false;
|
|
2629
3863
|
}
|
|
2630
3864
|
|
|
2631
3865
|
type DocumentAccessRecord = {
|
|
3866
|
+
id: string;
|
|
3867
|
+
authorityId: string | null;
|
|
2632
3868
|
authorityKind: string;
|
|
2633
3869
|
authorityWorkspaceId: string | null;
|
|
2634
3870
|
authoritySubjectId: string | null;
|
|
@@ -2765,7 +4001,8 @@ function mergeDocumentSearchRows(
|
|
|
2765
4001
|
right.score - left.score ||
|
|
2766
4002
|
(right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
|
|
2767
4003
|
(right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
|
|
2768
|
-
left.chunkIndex - right.chunkIndex
|
|
4004
|
+
left.chunkIndex - right.chunkIndex ||
|
|
4005
|
+
(left.chunkId === right.chunkId ? 0 : left.chunkId < right.chunkId ? -1 : 1),
|
|
2769
4006
|
);
|
|
2770
4007
|
}
|
|
2771
4008
|
|
|
@@ -3021,6 +4258,7 @@ function mapDocument(row: typeof schema.documents.$inferSelect): Document {
|
|
|
3021
4258
|
authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
|
|
3022
4259
|
authorityWorkspaceId: row.authorityWorkspaceId,
|
|
3023
4260
|
authoritySubjectId: row.authoritySubjectId,
|
|
4261
|
+
authorityId: row.authorityId,
|
|
3024
4262
|
visibility: normalizeDocumentVisibility(row.visibility),
|
|
3025
4263
|
createdBy: row.createdBy,
|
|
3026
4264
|
agentAccess: row.agentAccess,
|