@opengeni/documents 0.5.39 → 0.6.9-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +177 -5
- package/dist/index.js +881 -137
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/src/index.ts +1227 -150
package/src/index.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
KnowledgeProviderCitation,
|
|
4
|
+
type KnowledgeProviderCitation as KnowledgeProviderCitationValue,
|
|
5
|
+
type AddDocumentRequest,
|
|
4
6
|
CreateDocumentBaseRequest,
|
|
5
7
|
Document,
|
|
6
8
|
DocumentAuthorityKind,
|
|
9
|
+
DocumentAuthorityReclassification,
|
|
10
|
+
ListDocumentAuthorityReclassificationsResponse,
|
|
7
11
|
DocumentBase,
|
|
8
12
|
DocumentCuration,
|
|
9
13
|
DocumentCurationStatus,
|
|
@@ -15,12 +19,18 @@ import type {
|
|
|
15
19
|
IndexedDocumentSummary,
|
|
16
20
|
KnowledgeBrowseResponse,
|
|
17
21
|
KnowledgeRecord,
|
|
22
|
+
KnowledgeSearchResult,
|
|
18
23
|
KnowledgeSearchResponse,
|
|
19
24
|
KnowledgeSourceKind,
|
|
20
25
|
ListIndexedDocumentsResponse,
|
|
26
|
+
type ReclassifyDocumentAuthorityRequest,
|
|
27
|
+
type RunDocumentDefaultCollectionBackfillRequest,
|
|
28
|
+
DocumentDefaultCollectionBackfill,
|
|
21
29
|
} from "@opengeni/contracts";
|
|
22
30
|
import {
|
|
23
|
-
|
|
31
|
+
createPersonalDocumentAuthority,
|
|
32
|
+
getFilesForSubject,
|
|
33
|
+
resolveDocumentOriginalFileForSubject,
|
|
24
34
|
rlsContextForWorkspace,
|
|
25
35
|
setSubjectRlsContext,
|
|
26
36
|
withRlsContext,
|
|
@@ -29,11 +39,21 @@ import {
|
|
|
29
39
|
type Database,
|
|
30
40
|
} from "@opengeni/db";
|
|
31
41
|
import * as schema from "@opengeni/db/schema";
|
|
32
|
-
import type
|
|
33
|
-
import { createHash } from "node:crypto";
|
|
34
|
-
import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
|
|
42
|
+
import { retryWhileMissing, type ObjectStorage } from "@opengeni/storage";
|
|
43
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
44
|
+
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from "drizzle-orm";
|
|
35
45
|
import type OpenAI from "openai";
|
|
36
|
-
import {
|
|
46
|
+
import {
|
|
47
|
+
KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS,
|
|
48
|
+
KNOWLEDGE_BROWSE_MAX_LIMIT,
|
|
49
|
+
KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES,
|
|
50
|
+
KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES,
|
|
51
|
+
KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
|
|
52
|
+
KNOWLEDGE_SEARCH_MAX_RESULTS,
|
|
53
|
+
KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
|
|
54
|
+
KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
|
|
55
|
+
KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
56
|
+
} from "@opengeni/contracts";
|
|
37
57
|
import { projectKnowledgeRecord } from "./knowledge-projection";
|
|
38
58
|
|
|
39
59
|
export { projectKnowledgeRecord } from "./knowledge-projection";
|
|
@@ -136,6 +156,15 @@ export type DocumentAccessFilter = {
|
|
|
136
156
|
* only when the agent carries the creating subject as its viewer subject.
|
|
137
157
|
*/
|
|
138
158
|
agentOnly?: boolean | undefined;
|
|
159
|
+
/** Exact attempt whose grant snapshot must be revalidated in the content query. */
|
|
160
|
+
authorizedPersonalAttempt?:
|
|
161
|
+
| {
|
|
162
|
+
accountId: string;
|
|
163
|
+
workspaceId: string;
|
|
164
|
+
sessionId: string;
|
|
165
|
+
attemptId: string;
|
|
166
|
+
}
|
|
167
|
+
| undefined;
|
|
139
168
|
};
|
|
140
169
|
|
|
141
170
|
export type DocumentAuthority = {
|
|
@@ -144,6 +173,35 @@ export type DocumentAuthority = {
|
|
|
144
173
|
subjectId: string | null;
|
|
145
174
|
};
|
|
146
175
|
|
|
176
|
+
/** JSON-safe projection of the opaque capability minted by the API access layer. */
|
|
177
|
+
export type DocumentAccountAdminAuthorization = Readonly<{
|
|
178
|
+
authorizationId: string;
|
|
179
|
+
accountId: string;
|
|
180
|
+
actorSubjectId: string;
|
|
181
|
+
permission: "account:admin";
|
|
182
|
+
}>;
|
|
183
|
+
|
|
184
|
+
export type ReclassifyDocumentAuthorityInput = ReclassifyDocumentAuthorityRequest & {
|
|
185
|
+
accountId: string;
|
|
186
|
+
workspaceId: string;
|
|
187
|
+
documentId: string;
|
|
188
|
+
actorSubjectId: string;
|
|
189
|
+
accountAdminAuthorization: DocumentAccountAdminAuthorization | null;
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
export type RunDocumentDefaultCollectionBackfillInput =
|
|
193
|
+
RunDocumentDefaultCollectionBackfillRequest & {
|
|
194
|
+
accountId: string;
|
|
195
|
+
workspaceId: string;
|
|
196
|
+
actorSubjectId: string;
|
|
197
|
+
accountAdminAuthorization: DocumentAccountAdminAuthorization;
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
export type AgentDocumentAuthorityContext = {
|
|
201
|
+
sessionId: string;
|
|
202
|
+
attemptId: string;
|
|
203
|
+
};
|
|
204
|
+
|
|
147
205
|
export type DocumentInventoryStatusCounts = Record<DocumentStatus, number>;
|
|
148
206
|
export type DocumentInventorySourceKindCounts = Record<KnowledgeSourceKind, number>;
|
|
149
207
|
export type DocumentInventoryAuthorityKindCounts = Record<DocumentAuthorityKind, number>;
|
|
@@ -194,6 +252,7 @@ export type EffectiveDocumentSearchInput = Omit<DocumentSearchInput, "access"> &
|
|
|
194
252
|
initiatingSubjectId: string;
|
|
195
253
|
/** Agent retrieval additionally enforces documents.agent_access. */
|
|
196
254
|
surface: "human" | "agent";
|
|
255
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
197
256
|
};
|
|
198
257
|
|
|
199
258
|
export type ListEffectiveIndexedDocumentsInput = {
|
|
@@ -203,6 +262,7 @@ export type ListEffectiveIndexedDocumentsInput = {
|
|
|
203
262
|
initiatingSubjectId: string;
|
|
204
263
|
checkpoint?: string | undefined;
|
|
205
264
|
limit?: number | undefined;
|
|
265
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
206
266
|
};
|
|
207
267
|
|
|
208
268
|
export type EffectiveKnowledgeBrowseInput = {
|
|
@@ -210,12 +270,15 @@ export type EffectiveKnowledgeBrowseInput = {
|
|
|
210
270
|
workspaceId: string;
|
|
211
271
|
/** Immutable human subject accepted for the logical request/turn. */
|
|
212
272
|
initiatingSubjectId: string;
|
|
273
|
+
/** Human inspection bypasses agent_access but still uses exact subject/RLS authority. */
|
|
274
|
+
surface?: "human" | "agent" | undefined;
|
|
213
275
|
/** Omit to browse top-level documents; pass a document record id for chunks. */
|
|
214
276
|
parentId?: string | undefined;
|
|
215
277
|
topic?: string | undefined;
|
|
216
278
|
sourceKinds?: KnowledgeSourceKind[] | undefined;
|
|
217
279
|
cursor?: string | undefined;
|
|
218
280
|
limit?: number | undefined;
|
|
281
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
219
282
|
};
|
|
220
283
|
|
|
221
284
|
export type DocumentIndexHooks = {
|
|
@@ -929,6 +992,188 @@ export async function listDocumentBasesEnsuringDefault(
|
|
|
929
992
|
return await listDocumentBases(db, input.workspaceId);
|
|
930
993
|
}
|
|
931
994
|
|
|
995
|
+
export async function runDocumentDefaultCollectionBackfill(
|
|
996
|
+
db: Database,
|
|
997
|
+
input: RunDocumentDefaultCollectionBackfillInput,
|
|
998
|
+
) {
|
|
999
|
+
return await withDocumentAccountRls(
|
|
1000
|
+
db,
|
|
1001
|
+
input.accountId,
|
|
1002
|
+
input.workspaceId,
|
|
1003
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1004
|
+
async (scopedDb) => {
|
|
1005
|
+
const command = {
|
|
1006
|
+
accountId: input.accountId,
|
|
1007
|
+
workspaceId: input.workspaceId,
|
|
1008
|
+
actorSubjectId: input.actorSubjectId,
|
|
1009
|
+
runId: input.runId,
|
|
1010
|
+
operationId: input.operationId,
|
|
1011
|
+
batchSize: input.batchSize,
|
|
1012
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1013
|
+
};
|
|
1014
|
+
const rows = (await scopedDb.execute(sql`
|
|
1015
|
+
SELECT run_document_default_collection_backfill(
|
|
1016
|
+
${JSON.stringify(command)}::jsonb
|
|
1017
|
+
) AS result
|
|
1018
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1019
|
+
const row = rows[0];
|
|
1020
|
+
if (!row) throw new Error("Document Default collection backfill returned no result");
|
|
1021
|
+
return DocumentDefaultCollectionBackfill.parse(row.result);
|
|
1022
|
+
},
|
|
1023
|
+
);
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
export async function reclassifyDocumentAuthority(
|
|
1027
|
+
db: Database,
|
|
1028
|
+
input: ReclassifyDocumentAuthorityInput,
|
|
1029
|
+
) {
|
|
1030
|
+
return await withDocumentAccountRls(
|
|
1031
|
+
db,
|
|
1032
|
+
input.accountId,
|
|
1033
|
+
input.workspaceId,
|
|
1034
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1035
|
+
async (scopedDb) => {
|
|
1036
|
+
const command = {
|
|
1037
|
+
accountId: input.accountId,
|
|
1038
|
+
workspaceId: input.workspaceId,
|
|
1039
|
+
documentId: input.documentId,
|
|
1040
|
+
operationId: input.operationId,
|
|
1041
|
+
actorSubjectId: input.actorSubjectId,
|
|
1042
|
+
expectedAuthority: input.expectedAuthority,
|
|
1043
|
+
targetAuthorityKind: input.targetAuthorityKind,
|
|
1044
|
+
accountAdminAuthorization: input.accountAdminAuthorization,
|
|
1045
|
+
};
|
|
1046
|
+
const rows = (await scopedDb.execute(sql`
|
|
1047
|
+
SELECT reclassify_document_authority(${JSON.stringify(command)}::jsonb) AS result
|
|
1048
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1049
|
+
const row = rows[0];
|
|
1050
|
+
if (!row) throw new Error("Document authority reclassification returned no result");
|
|
1051
|
+
return DocumentAuthorityReclassification.parse(row.result);
|
|
1052
|
+
},
|
|
1053
|
+
);
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
export async function listDocumentAuthorityReclassifications(
|
|
1057
|
+
db: Database,
|
|
1058
|
+
input: {
|
|
1059
|
+
accountId: string;
|
|
1060
|
+
workspaceId: string;
|
|
1061
|
+
documentId: string;
|
|
1062
|
+
actorSubjectId: string;
|
|
1063
|
+
limit?: number | undefined;
|
|
1064
|
+
cursor?: string | undefined;
|
|
1065
|
+
},
|
|
1066
|
+
) {
|
|
1067
|
+
const limit = input.limit ?? 50;
|
|
1068
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
|
|
1069
|
+
throw new Error("document authority receipt limit must be between 1 and 100");
|
|
1070
|
+
}
|
|
1071
|
+
const cursor = input.cursor
|
|
1072
|
+
? decodeDocumentAuthorityReclassificationCursor(input.cursor, input)
|
|
1073
|
+
: null;
|
|
1074
|
+
return await withDocumentAccountRls(
|
|
1075
|
+
db,
|
|
1076
|
+
input.accountId,
|
|
1077
|
+
input.workspaceId,
|
|
1078
|
+
{ viewerSubjectId: input.actorSubjectId },
|
|
1079
|
+
async (scopedDb) => {
|
|
1080
|
+
const rows = (await scopedDb.execute(sql`
|
|
1081
|
+
SELECT list_document_authority_reclassifications(
|
|
1082
|
+
${input.accountId}::uuid,
|
|
1083
|
+
${input.workspaceId}::uuid,
|
|
1084
|
+
${input.actorSubjectId},
|
|
1085
|
+
${input.documentId}::uuid,
|
|
1086
|
+
${limit + 1}::integer,
|
|
1087
|
+
${cursor?.createdAt ?? null}::timestamptz,
|
|
1088
|
+
${cursor?.operationId ?? null}::uuid
|
|
1089
|
+
) AS result
|
|
1090
|
+
`)) as unknown as Array<{ result: unknown }>;
|
|
1091
|
+
const parsed = rows.map((row) => DocumentAuthorityReclassification.parse(row.result));
|
|
1092
|
+
const hasMore = parsed.length > limit;
|
|
1093
|
+
const receipts = parsed.slice(0, limit);
|
|
1094
|
+
const tail = hasMore ? receipts.at(-1) : null;
|
|
1095
|
+
return ListDocumentAuthorityReclassificationsResponse.parse({
|
|
1096
|
+
receipts,
|
|
1097
|
+
hasMore,
|
|
1098
|
+
nextCursor: tail
|
|
1099
|
+
? encodeDocumentAuthorityReclassificationCursor(input, {
|
|
1100
|
+
createdAt: tail.createdAt,
|
|
1101
|
+
operationId: tail.operationId,
|
|
1102
|
+
})
|
|
1103
|
+
: null,
|
|
1104
|
+
});
|
|
1105
|
+
},
|
|
1106
|
+
);
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
type DocumentAuthorityReclassificationCursor = {
|
|
1110
|
+
createdAt: string;
|
|
1111
|
+
operationId: string;
|
|
1112
|
+
};
|
|
1113
|
+
|
|
1114
|
+
function documentAuthorityReclassificationCursorScope(input: {
|
|
1115
|
+
accountId: string;
|
|
1116
|
+
workspaceId: string;
|
|
1117
|
+
documentId: string;
|
|
1118
|
+
actorSubjectId: string;
|
|
1119
|
+
}): string {
|
|
1120
|
+
return createHash("sha256")
|
|
1121
|
+
.update(
|
|
1122
|
+
JSON.stringify([
|
|
1123
|
+
"document_authority_reclassification_cursor",
|
|
1124
|
+
1,
|
|
1125
|
+
input.accountId,
|
|
1126
|
+
input.workspaceId,
|
|
1127
|
+
input.documentId,
|
|
1128
|
+
input.actorSubjectId,
|
|
1129
|
+
]),
|
|
1130
|
+
"utf8",
|
|
1131
|
+
)
|
|
1132
|
+
.digest("hex")
|
|
1133
|
+
.slice(0, 32);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
function encodeDocumentAuthorityReclassificationCursor(
|
|
1137
|
+
scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
|
|
1138
|
+
cursor: DocumentAuthorityReclassificationCursor,
|
|
1139
|
+
): string {
|
|
1140
|
+
return Buffer.from(
|
|
1141
|
+
JSON.stringify({
|
|
1142
|
+
v: 1,
|
|
1143
|
+
s: documentAuthorityReclassificationCursorScope(scope),
|
|
1144
|
+
t: cursor.createdAt,
|
|
1145
|
+
i: cursor.operationId,
|
|
1146
|
+
}),
|
|
1147
|
+
"utf8",
|
|
1148
|
+
).toString("base64url");
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
function decodeDocumentAuthorityReclassificationCursor(
|
|
1152
|
+
value: string,
|
|
1153
|
+
scope: Parameters<typeof documentAuthorityReclassificationCursorScope>[0],
|
|
1154
|
+
): DocumentAuthorityReclassificationCursor {
|
|
1155
|
+
try {
|
|
1156
|
+
if (!value || value.length > 1_024) throw new Error("cursor length");
|
|
1157
|
+
const bytes = Buffer.from(value, "base64url");
|
|
1158
|
+
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
1159
|
+
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
1160
|
+
if (
|
|
1161
|
+
Object.keys(parsed).sort().join(",") !== "i,s,t,v" ||
|
|
1162
|
+
parsed.v !== 1 ||
|
|
1163
|
+
parsed.s !== documentAuthorityReclassificationCursorScope(scope) ||
|
|
1164
|
+
typeof parsed.t !== "string" ||
|
|
1165
|
+
!Number.isFinite(new Date(parsed.t).getTime()) ||
|
|
1166
|
+
typeof parsed.i !== "string" ||
|
|
1167
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(parsed.i)
|
|
1168
|
+
) {
|
|
1169
|
+
throw new Error("cursor payload");
|
|
1170
|
+
}
|
|
1171
|
+
return { createdAt: parsed.t, operationId: parsed.i };
|
|
1172
|
+
} catch (error) {
|
|
1173
|
+
throw new Error("invalid document authority receipt cursor", { cause: error });
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
|
|
932
1177
|
export async function addDocumentToBase(
|
|
933
1178
|
db: Database,
|
|
934
1179
|
input: AddDocumentRequest & {
|
|
@@ -967,7 +1212,18 @@ export async function addDocumentToBase(
|
|
|
967
1212
|
if (viewerSubjectId) await setSubjectRlsContext(scopedDb, viewerSubjectId);
|
|
968
1213
|
const base = await getDocumentBase(scopedDb, input.workspaceId, input.baseId);
|
|
969
1214
|
if (!base) throw new Error(`Document base not found: ${input.baseId}`);
|
|
970
|
-
const
|
|
1215
|
+
const initiatingSubjectId = cleanString(input.initiatingSubjectId ?? null);
|
|
1216
|
+
const createdBy = cleanString(input.createdBy ?? null);
|
|
1217
|
+
if (initiatingSubjectId && createdBy && initiatingSubjectId !== createdBy) {
|
|
1218
|
+
throw new Error("document file authority must match the exact initiating subject");
|
|
1219
|
+
}
|
|
1220
|
+
const fileAuthoritySubjectId = initiatingSubjectId ?? createdBy ?? null;
|
|
1221
|
+
const file = await requireReadyFile(scopedDb, {
|
|
1222
|
+
accountId: input.accountId,
|
|
1223
|
+
workspaceId: input.workspaceId,
|
|
1224
|
+
subjectId: fileAuthoritySubjectId,
|
|
1225
|
+
fileId: input.fileId,
|
|
1226
|
+
});
|
|
971
1227
|
const knowledgeSourceIdentity = cleanString(input.knowledgeSourceIdentity ?? null);
|
|
972
1228
|
if (knowledgeSourceIdentity && knowledgeSourceIdentity.length > 512) {
|
|
973
1229
|
throw new Error("knowledge source document identity exceeds 512 characters");
|
|
@@ -1029,36 +1285,53 @@ export async function addDocumentToBase(
|
|
|
1029
1285
|
.returning();
|
|
1030
1286
|
return mapDocument(updated ?? existing);
|
|
1031
1287
|
}
|
|
1032
|
-
const
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1288
|
+
const documentId = randomUUID();
|
|
1289
|
+
const row = await scopedDb.transaction(async (tx) => {
|
|
1290
|
+
const userAuthority =
|
|
1291
|
+
authority.kind === "personal"
|
|
1292
|
+
? await createPersonalDocumentAuthority(tx, {
|
|
1293
|
+
accountId: input.accountId,
|
|
1294
|
+
workspaceId: input.workspaceId,
|
|
1295
|
+
subjectId: authority.subjectId!,
|
|
1296
|
+
documentId,
|
|
1297
|
+
})
|
|
1298
|
+
: null;
|
|
1299
|
+
const [inserted] = await tx
|
|
1300
|
+
.insert(schema.documents)
|
|
1301
|
+
.values({
|
|
1302
|
+
id: documentId,
|
|
1303
|
+
accountId: input.accountId,
|
|
1304
|
+
workspaceId: input.workspaceId,
|
|
1305
|
+
baseId: input.baseId,
|
|
1306
|
+
fileId: input.fileId,
|
|
1307
|
+
status: "queued",
|
|
1308
|
+
title: cleanString(input.title) ?? cleanString(input.sourceTitle) ?? file.filename,
|
|
1309
|
+
parser: DEFAULT_DOCUMENT_PARSER,
|
|
1310
|
+
sourceKind: input.sourceKind ?? "manual_upload",
|
|
1311
|
+
sourceUri: cleanString(input.sourceUri) ?? null,
|
|
1312
|
+
sourceExternalId: cleanString(input.sourceExternalId) ?? null,
|
|
1313
|
+
sourceTitle: cleanString(input.sourceTitle) ?? null,
|
|
1314
|
+
sourceAuthor: cleanString(input.sourceAuthor) ?? null,
|
|
1315
|
+
sourceCreatedAt: parseOptionalDate(input.sourceCreatedAt),
|
|
1316
|
+
sourceUpdatedAt: parseOptionalDate(input.sourceUpdatedAt),
|
|
1317
|
+
sourceVersion: cleanString(input.sourceVersion) ?? null,
|
|
1318
|
+
knowledgeSourceIdentity,
|
|
1319
|
+
aclTags: cleanStringArray(input.aclTags),
|
|
1320
|
+
authorityKind: authority.kind,
|
|
1321
|
+
authorityWorkspaceId: userAuthority ? null : authority.workspaceId,
|
|
1322
|
+
authoritySubjectId: authority.subjectId,
|
|
1323
|
+
authorityId: userAuthority?.authorityId ?? null,
|
|
1324
|
+
ownerOrganizationMembershipId: userAuthority?.ownerOrganizationMembershipId ?? null,
|
|
1325
|
+
originWorkspaceId: input.workspaceId,
|
|
1326
|
+
visibility: authority.kind === "personal" ? "private" : "workspace",
|
|
1327
|
+
agentAccess: input.agentAccess ?? true,
|
|
1328
|
+
createdBy: fileAuthoritySubjectId,
|
|
1329
|
+
curationStatus: input.curationStatus ?? "none",
|
|
1330
|
+
updatedAt: now,
|
|
1331
|
+
})
|
|
1332
|
+
.returning();
|
|
1333
|
+
return inserted;
|
|
1334
|
+
});
|
|
1062
1335
|
if (!row) throw new Error("Failed to create document");
|
|
1063
1336
|
return mapDocument(row);
|
|
1064
1337
|
},
|
|
@@ -1092,7 +1365,7 @@ export async function moveDocumentToBase(
|
|
|
1092
1365
|
.from(schema.documents)
|
|
1093
1366
|
.where(
|
|
1094
1367
|
and(
|
|
1095
|
-
eq(schema.documents.
|
|
1368
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1096
1369
|
eq(schema.documents.id, input.documentId),
|
|
1097
1370
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1098
1371
|
),
|
|
@@ -1107,14 +1380,18 @@ export async function moveDocumentToBase(
|
|
|
1107
1380
|
throw new Error("document has no suggested base; pass targetBaseId");
|
|
1108
1381
|
}
|
|
1109
1382
|
if (targetBaseId === row.baseId) return mapDocument(row);
|
|
1110
|
-
|
|
1383
|
+
// A portable personal Document retains its immutable ingestion workspace
|
|
1384
|
+
// as provenance and physical storage. Management may be authorized from
|
|
1385
|
+
// another same-organization workspace, but filing still targets a base
|
|
1386
|
+
// in that origin workspace rather than silently copying authority/data.
|
|
1387
|
+
const base = await getDocumentBase(scopedDb, row.workspaceId, targetBaseId);
|
|
1111
1388
|
if (!base) throw new Error(`Document base not found: ${targetBaseId}`);
|
|
1112
1389
|
const [conflict] = await scopedDb
|
|
1113
1390
|
.select({ id: schema.documents.id })
|
|
1114
1391
|
.from(schema.documents)
|
|
1115
1392
|
.where(
|
|
1116
1393
|
and(
|
|
1117
|
-
eq(schema.documents.workspaceId,
|
|
1394
|
+
eq(schema.documents.workspaceId, row.workspaceId),
|
|
1118
1395
|
eq(schema.documents.baseId, targetBaseId),
|
|
1119
1396
|
...(row.knowledgeSourceIdentity
|
|
1120
1397
|
? [eq(schema.documents.knowledgeSourceIdentity, row.knowledgeSourceIdentity)]
|
|
@@ -1138,7 +1415,7 @@ export async function moveDocumentToBase(
|
|
|
1138
1415
|
})
|
|
1139
1416
|
.where(
|
|
1140
1417
|
and(
|
|
1141
|
-
eq(schema.documents.workspaceId,
|
|
1418
|
+
eq(schema.documents.workspaceId, row.workspaceId),
|
|
1142
1419
|
eq(schema.documents.id, input.documentId),
|
|
1143
1420
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1144
1421
|
),
|
|
@@ -1150,7 +1427,7 @@ export async function moveDocumentToBase(
|
|
|
1150
1427
|
.set({ baseId: targetBaseId })
|
|
1151
1428
|
.where(
|
|
1152
1429
|
and(
|
|
1153
|
-
eq(schema.documentChunks.workspaceId,
|
|
1430
|
+
eq(schema.documentChunks.workspaceId, row.workspaceId),
|
|
1154
1431
|
eq(schema.documentChunks.documentId, input.documentId),
|
|
1155
1432
|
),
|
|
1156
1433
|
);
|
|
@@ -1185,7 +1462,7 @@ export async function deleteDocumentFromBase(
|
|
|
1185
1462
|
.from(schema.documents)
|
|
1186
1463
|
.where(
|
|
1187
1464
|
and(
|
|
1188
|
-
eq(schema.documents.
|
|
1465
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1189
1466
|
eq(schema.documents.id, input.documentId),
|
|
1190
1467
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1191
1468
|
),
|
|
@@ -1205,7 +1482,7 @@ export async function deleteDocumentFromBase(
|
|
|
1205
1482
|
.delete(schema.documents)
|
|
1206
1483
|
.where(
|
|
1207
1484
|
and(
|
|
1208
|
-
eq(schema.documents.
|
|
1485
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1209
1486
|
eq(schema.documents.id, input.documentId),
|
|
1210
1487
|
...documentAccessConditions(input.workspaceId, input.access),
|
|
1211
1488
|
),
|
|
@@ -1236,6 +1513,28 @@ export async function listDocuments(
|
|
|
1236
1513
|
});
|
|
1237
1514
|
}
|
|
1238
1515
|
|
|
1516
|
+
/**
|
|
1517
|
+
* List Documents the human can discover and manage from the requested
|
|
1518
|
+
* workspace. Organization Documents are account-wide, workspace Documents are
|
|
1519
|
+
* local, activated organization-user Documents are portable across the owner's
|
|
1520
|
+
* same-organization workspaces, and legacy personal rows remain anchored to
|
|
1521
|
+
* their ingestion workspace through documentAccessConditions.
|
|
1522
|
+
*/
|
|
1523
|
+
export async function listAccessibleDocuments(
|
|
1524
|
+
db: Database,
|
|
1525
|
+
workspaceId: string,
|
|
1526
|
+
access: DocumentAccessFilter,
|
|
1527
|
+
): Promise<Document[]> {
|
|
1528
|
+
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
1529
|
+
const rows = await scopedDb
|
|
1530
|
+
.select()
|
|
1531
|
+
.from(schema.documents)
|
|
1532
|
+
.where(and(...documentAccessConditions(workspaceId, access)))
|
|
1533
|
+
.orderBy(desc(schema.documents.updatedAt), asc(schema.documents.createdAt));
|
|
1534
|
+
return rows.map(mapDocument);
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1239
1538
|
/**
|
|
1240
1539
|
* List newly ready documents in the same effective scope used by agent
|
|
1241
1540
|
* retrieval. The opaque checkpoint is bound to the account, requesting
|
|
@@ -1258,10 +1557,13 @@ export async function listEffectiveIndexedDocuments(
|
|
|
1258
1557
|
initiatingSubjectId,
|
|
1259
1558
|
})
|
|
1260
1559
|
: 0n;
|
|
1261
|
-
const access
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1560
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
1561
|
+
accountId: input.accountId,
|
|
1562
|
+
workspaceId: input.workspaceId,
|
|
1563
|
+
initiatingSubjectId,
|
|
1564
|
+
surface: "agent",
|
|
1565
|
+
agentAuthority: input.agentAuthority,
|
|
1566
|
+
});
|
|
1265
1567
|
const rows = await withDocumentAccountRls(
|
|
1266
1568
|
db,
|
|
1267
1569
|
input.accountId,
|
|
@@ -1392,11 +1694,55 @@ export async function getDocument(
|
|
|
1392
1694
|
.select()
|
|
1393
1695
|
.from(schema.documents)
|
|
1394
1696
|
.where(
|
|
1395
|
-
and(
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1697
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1698
|
+
)
|
|
1699
|
+
.limit(1);
|
|
1700
|
+
return row ? mapDocument(row) : null;
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
/**
|
|
1705
|
+
* Resolve the immutable source file through Document authority in the requested
|
|
1706
|
+
* workspace. The file remains physically owned by its ingestion workspace;
|
|
1707
|
+
* callers never gain generic access to that workspace's file inventory.
|
|
1708
|
+
*/
|
|
1709
|
+
export async function getDocumentOriginalFile(
|
|
1710
|
+
db: Database,
|
|
1711
|
+
input: {
|
|
1712
|
+
accountId: string;
|
|
1713
|
+
workspaceId: string;
|
|
1714
|
+
documentId: string;
|
|
1715
|
+
access: DocumentAccessFilter;
|
|
1716
|
+
},
|
|
1717
|
+
): Promise<FileAsset | null> {
|
|
1718
|
+
const subjectId = cleanString(input.access.viewerSubjectId ?? null);
|
|
1719
|
+
if (!subjectId || input.access.agentOnly) return null;
|
|
1720
|
+
return await resolveDocumentOriginalFileForSubject(db, {
|
|
1721
|
+
accountId: input.accountId,
|
|
1722
|
+
workspaceId: input.workspaceId,
|
|
1723
|
+
subjectId,
|
|
1724
|
+
documentId: input.documentId,
|
|
1725
|
+
});
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/**
|
|
1729
|
+
* Internal ingestion-only read used after the worker has independently resolved
|
|
1730
|
+
* and fenced the immutable document authority tuple. It deliberately does not
|
|
1731
|
+
* apply provider retrieval authorization because a new Drive document must be
|
|
1732
|
+
* indexed before its first ACL evidence can be attached. User, API, MCP, and
|
|
1733
|
+
* agent reads must use getDocument/effective retrieval instead.
|
|
1734
|
+
*/
|
|
1735
|
+
export async function getDocumentForIndexing(
|
|
1736
|
+
db: Database,
|
|
1737
|
+
workspaceId: string,
|
|
1738
|
+
documentId: string,
|
|
1739
|
+
): Promise<Document | null> {
|
|
1740
|
+
return await withWorkspaceRls(db, workspaceId, async (scopedDb) => {
|
|
1741
|
+
const [row] = await scopedDb
|
|
1742
|
+
.select()
|
|
1743
|
+
.from(schema.documents)
|
|
1744
|
+
.where(
|
|
1745
|
+
and(eq(schema.documents.workspaceId, workspaceId), eq(schema.documents.id, documentId)),
|
|
1400
1746
|
)
|
|
1401
1747
|
.limit(1);
|
|
1402
1748
|
return row ? mapDocument(row) : null;
|
|
@@ -1412,14 +1758,10 @@ export async function queueDocumentForReindex(
|
|
|
1412
1758
|
): Promise<Document> {
|
|
1413
1759
|
return await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
1414
1760
|
const [document] = await scopedDb
|
|
1415
|
-
.select(
|
|
1761
|
+
.select()
|
|
1416
1762
|
.from(schema.documents)
|
|
1417
1763
|
.where(
|
|
1418
|
-
and(
|
|
1419
|
-
eq(schema.documents.workspaceId, workspaceId),
|
|
1420
|
-
eq(schema.documents.id, documentId),
|
|
1421
|
-
...documentAccessConditions(workspaceId, access),
|
|
1422
|
-
),
|
|
1764
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1423
1765
|
)
|
|
1424
1766
|
.limit(1);
|
|
1425
1767
|
if (!document) throw new Error(`Document not found: ${documentId}`);
|
|
@@ -1432,11 +1774,7 @@ export async function queueDocumentForReindex(
|
|
|
1432
1774
|
updatedAt: new Date(),
|
|
1433
1775
|
})
|
|
1434
1776
|
.where(
|
|
1435
|
-
and(
|
|
1436
|
-
eq(schema.documents.workspaceId, workspaceId),
|
|
1437
|
-
eq(schema.documents.id, documentId),
|
|
1438
|
-
...documentAccessConditions(workspaceId, access),
|
|
1439
|
-
),
|
|
1777
|
+
and(eq(schema.documents.id, documentId), ...documentAccessConditions(workspaceId, access)),
|
|
1440
1778
|
)
|
|
1441
1779
|
.returning();
|
|
1442
1780
|
if (!row) throw new Error(`Document not found: ${documentId}`);
|
|
@@ -1477,7 +1815,12 @@ export async function indexDocumentNow(
|
|
|
1477
1815
|
);
|
|
1478
1816
|
if (!loadedDocument) throw new Error(`Document not found: ${documentId}`);
|
|
1479
1817
|
let document: DocumentRow = loadedDocument;
|
|
1480
|
-
const file = await requireReadyFile(db,
|
|
1818
|
+
const file = await requireReadyFile(db, {
|
|
1819
|
+
accountId: document.accountId,
|
|
1820
|
+
workspaceId,
|
|
1821
|
+
subjectId: cleanString(document.createdBy) ?? null,
|
|
1822
|
+
fileId: document.fileId,
|
|
1823
|
+
});
|
|
1481
1824
|
await withDocumentRls(db, workspaceId, access, async (scopedDb) => {
|
|
1482
1825
|
await scopedDb
|
|
1483
1826
|
.update(schema.documents)
|
|
@@ -1492,7 +1835,11 @@ export async function indexDocumentNow(
|
|
|
1492
1835
|
);
|
|
1493
1836
|
});
|
|
1494
1837
|
try {
|
|
1495
|
-
const
|
|
1838
|
+
const object = await retryWhileMissing(async () =>
|
|
1839
|
+
objectStorage.getObjectBytes(file.objectKey),
|
|
1840
|
+
);
|
|
1841
|
+
if (!object) throw new Error("document source object is missing");
|
|
1842
|
+
const bytes = object.bytes;
|
|
1496
1843
|
const parsed = await services.parser.parse(bytes, file);
|
|
1497
1844
|
// Knowledge drops (curationStatus 'pending') are curated between parse and
|
|
1498
1845
|
// chunking when a provider is enabled, so chunk metadata and base placement
|
|
@@ -1604,9 +1951,7 @@ export async function indexDocumentNow(
|
|
|
1604
1951
|
// Internal indexing must be able to return a private document to the caller
|
|
1605
1952
|
// that created/queued it. Public reads remain fail-closed when no subject is
|
|
1606
1953
|
// supplied; the creator subject is the document's frozen access principal.
|
|
1607
|
-
const updated = await
|
|
1608
|
-
viewerSubjectId: document.authoritySubjectId,
|
|
1609
|
-
});
|
|
1954
|
+
const updated = await getDocumentForIndexing(db, workspaceId, documentId);
|
|
1610
1955
|
if (!updated) throw new Error(`Document disappeared after indexing: ${documentId}`);
|
|
1611
1956
|
return updated;
|
|
1612
1957
|
}
|
|
@@ -1750,6 +2095,20 @@ export async function searchDocuments(
|
|
|
1750
2095
|
input: DocumentSearchInput,
|
|
1751
2096
|
services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
|
|
1752
2097
|
): Promise<DocumentSearchResult[]> {
|
|
2098
|
+
return (await searchDocumentCandidates(db, input, services)).results;
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
export type DocumentCandidateRelevanceFloor = {
|
|
2102
|
+
vectorScore: number;
|
|
2103
|
+
keywordScore: number;
|
|
2104
|
+
};
|
|
2105
|
+
|
|
2106
|
+
async function searchDocumentCandidates(
|
|
2107
|
+
db: Database,
|
|
2108
|
+
input: DocumentSearchInput,
|
|
2109
|
+
services: Pick<DocumentServices, "embedder">,
|
|
2110
|
+
relevanceFloor?: DocumentCandidateRelevanceFloor,
|
|
2111
|
+
): Promise<{ results: DocumentSearchResult[]; belowRelevanceFloor: number }> {
|
|
1753
2112
|
await assertDocumentAccountWorkspace(db, input.accountId, input.workspaceId);
|
|
1754
2113
|
const mode = input.mode ?? "hybrid";
|
|
1755
2114
|
const limit = Math.min(Math.max(input.limit ?? 5, 1), 50);
|
|
@@ -1774,7 +2133,28 @@ export async function searchDocuments(
|
|
|
1774
2133
|
if (mode === "keyword" || mode === "hybrid") {
|
|
1775
2134
|
rows.push(...(await keywordSearchDocuments(db, input, candidateLimit)));
|
|
1776
2135
|
}
|
|
1777
|
-
|
|
2136
|
+
const merged = mergeDocumentSearchRows(rows, mode);
|
|
2137
|
+
return selectDocumentSearchCandidateWindow(merged, limit, relevanceFloor);
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
export function selectDocumentSearchCandidateWindow(
|
|
2141
|
+
merged: DocumentSearchResult[],
|
|
2142
|
+
limit: number,
|
|
2143
|
+
relevanceFloor?: DocumentCandidateRelevanceFloor,
|
|
2144
|
+
): { results: DocumentSearchResult[]; belowRelevanceFloor: number } {
|
|
2145
|
+
const boundedLimit = Math.min(Math.max(Math.trunc(limit), 1), 50);
|
|
2146
|
+
if (!relevanceFloor) {
|
|
2147
|
+
return { results: merged.slice(0, boundedLimit), belowRelevanceFloor: 0 };
|
|
2148
|
+
}
|
|
2149
|
+
let belowRelevanceFloor = 0;
|
|
2150
|
+
const relevant = merged.filter((result) => {
|
|
2151
|
+
const included =
|
|
2152
|
+
(result.vectorScore !== null && result.vectorScore >= relevanceFloor.vectorScore) ||
|
|
2153
|
+
(result.keywordScore !== null && result.keywordScore >= relevanceFloor.keywordScore);
|
|
2154
|
+
if (!included) belowRelevanceFloor += 1;
|
|
2155
|
+
return included;
|
|
2156
|
+
});
|
|
2157
|
+
return { results: relevant.slice(0, boundedLimit), belowRelevanceFloor };
|
|
1778
2158
|
}
|
|
1779
2159
|
|
|
1780
2160
|
/**
|
|
@@ -1791,6 +2171,13 @@ export async function searchEffectiveDocuments(
|
|
|
1791
2171
|
if (!initiatingSubjectId) {
|
|
1792
2172
|
throw new Error("effective document retrieval requires an initiating subject");
|
|
1793
2173
|
}
|
|
2174
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2175
|
+
accountId: input.accountId,
|
|
2176
|
+
workspaceId: input.workspaceId,
|
|
2177
|
+
initiatingSubjectId,
|
|
2178
|
+
surface: input.surface,
|
|
2179
|
+
agentAuthority: input.agentAuthority,
|
|
2180
|
+
});
|
|
1794
2181
|
return await searchDocuments(
|
|
1795
2182
|
db,
|
|
1796
2183
|
{
|
|
@@ -1804,10 +2191,7 @@ export async function searchEffectiveDocuments(
|
|
|
1804
2191
|
aclTags: input.aclTags,
|
|
1805
2192
|
// Construct the lower-level access filter here instead of spreading the
|
|
1806
2193
|
// caller input, so an untyped/legacy access override is always ignored.
|
|
1807
|
-
access
|
|
1808
|
-
viewerSubjectId: initiatingSubjectId,
|
|
1809
|
-
...(input.surface === "agent" ? { agentOnly: true } : {}),
|
|
1810
|
-
},
|
|
2194
|
+
access,
|
|
1811
2195
|
},
|
|
1812
2196
|
services,
|
|
1813
2197
|
);
|
|
@@ -1825,13 +2209,45 @@ export async function searchEffectiveKnowledge(
|
|
|
1825
2209
|
services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
|
|
1826
2210
|
): Promise<KnowledgeSearchResponse> {
|
|
1827
2211
|
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1828
|
-
const
|
|
2212
|
+
const requestedLimit = Math.min(Math.max(input.limit ?? 5, 1), KNOWLEDGE_SEARCH_MAX_RESULTS);
|
|
2213
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2214
|
+
accountId: input.accountId,
|
|
2215
|
+
workspaceId: input.workspaceId,
|
|
2216
|
+
initiatingSubjectId,
|
|
2217
|
+
surface: input.surface,
|
|
2218
|
+
agentAuthority: input.agentAuthority,
|
|
2219
|
+
});
|
|
2220
|
+
// Pull a bounded surplus so the permission-safe result set can still satisfy
|
|
2221
|
+
// the caller after relevance filtering and exact-content deduplication.
|
|
2222
|
+
const candidateLimit = Math.min(requestedLimit * 4, KNOWLEDGE_SEARCH_MAX_RESULTS);
|
|
2223
|
+
const rankedSelection = await searchDocumentCandidates(
|
|
1829
2224
|
db,
|
|
1830
|
-
{
|
|
2225
|
+
{
|
|
2226
|
+
accountId: input.accountId,
|
|
2227
|
+
workspaceId: input.workspaceId,
|
|
2228
|
+
query: input.query,
|
|
2229
|
+
limit: candidateLimit,
|
|
2230
|
+
...(input.baseIds ? { baseIds: input.baseIds } : {}),
|
|
2231
|
+
...(input.mode ? { mode: input.mode } : {}),
|
|
2232
|
+
...(input.sourceKinds ? { sourceKinds: input.sourceKinds } : {}),
|
|
2233
|
+
...(input.aclTags ? { aclTags: input.aclTags } : {}),
|
|
2234
|
+
access,
|
|
2235
|
+
},
|
|
1831
2236
|
services,
|
|
2237
|
+
{
|
|
2238
|
+
vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE,
|
|
2239
|
+
keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE,
|
|
2240
|
+
},
|
|
1832
2241
|
);
|
|
1833
|
-
|
|
1834
|
-
|
|
2242
|
+
const ranked = rankedSelection.results;
|
|
2243
|
+
if (ranked.length === 0) {
|
|
2244
|
+
return selectKnowledgeSearchResults({
|
|
2245
|
+
candidates: [],
|
|
2246
|
+
rankedCandidateCount: 0,
|
|
2247
|
+
requestedLimit,
|
|
2248
|
+
alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
|
|
2249
|
+
});
|
|
2250
|
+
}
|
|
1835
2251
|
const current = await withDocumentAccountRls(
|
|
1836
2252
|
db,
|
|
1837
2253
|
input.accountId,
|
|
@@ -1839,7 +2255,13 @@ export async function searchEffectiveKnowledge(
|
|
|
1839
2255
|
access,
|
|
1840
2256
|
async (scopedDb) =>
|
|
1841
2257
|
await scopedDb
|
|
1842
|
-
.select({
|
|
2258
|
+
.select({
|
|
2259
|
+
chunk: schema.documentChunks,
|
|
2260
|
+
document: schema.documents,
|
|
2261
|
+
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2262
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
2263
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
2264
|
+
})
|
|
1843
2265
|
.from(schema.documentChunks)
|
|
1844
2266
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1845
2267
|
.where(
|
|
@@ -1856,23 +2278,399 @@ export async function searchEffectiveKnowledge(
|
|
|
1856
2278
|
),
|
|
1857
2279
|
);
|
|
1858
2280
|
const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
|
|
1859
|
-
return {
|
|
1860
|
-
|
|
2281
|
+
return selectKnowledgeSearchResults({
|
|
2282
|
+
rankedCandidateCount: ranked.length,
|
|
2283
|
+
requestedLimit,
|
|
2284
|
+
alreadyBelowRelevanceFloor: rankedSelection.belowRelevanceFloor,
|
|
2285
|
+
candidates: ranked.flatMap((rankedResult) => {
|
|
1861
2286
|
const row = currentByChunkId.get(rankedResult.chunkId);
|
|
1862
2287
|
if (!row) return [];
|
|
1863
2288
|
return [
|
|
1864
2289
|
{
|
|
1865
|
-
record: knowledgeChunkRecord(row.document, row.chunk
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
2290
|
+
record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
2291
|
+
previousChunkId: row.previousChunkId,
|
|
2292
|
+
nextChunkId: row.nextChunkId,
|
|
2293
|
+
}),
|
|
2294
|
+
semanticScore: rankedResult.score,
|
|
2295
|
+
matchType: rankedResult.matchType,
|
|
2296
|
+
vectorScore: rankedResult.vectorScore,
|
|
2297
|
+
keywordScore: rankedResult.keywordScore,
|
|
1872
2298
|
},
|
|
1873
2299
|
];
|
|
1874
2300
|
}),
|
|
2301
|
+
});
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
type KnowledgeSearchCandidate = {
|
|
2305
|
+
record: KnowledgeRecord;
|
|
2306
|
+
semanticScore: number;
|
|
2307
|
+
matchType: DocumentSearchMode;
|
|
2308
|
+
vectorScore: number | null;
|
|
2309
|
+
keywordScore: number | null;
|
|
2310
|
+
};
|
|
2311
|
+
|
|
2312
|
+
type KnowledgeSearchSelectionInput = {
|
|
2313
|
+
candidates: KnowledgeSearchCandidate[];
|
|
2314
|
+
rankedCandidateCount: number;
|
|
2315
|
+
requestedLimit: number;
|
|
2316
|
+
alreadyBelowRelevanceFloor?: number | undefined;
|
|
2317
|
+
now?: Date | undefined;
|
|
2318
|
+
};
|
|
2319
|
+
|
|
2320
|
+
/**
|
|
2321
|
+
* Deterministic, content-safe final selection over already-authorized and
|
|
2322
|
+
* freshly rechecked Knowledge candidates. Exported for boundary tests; callers
|
|
2323
|
+
* must never use it as a substitute for the database authorization pass above.
|
|
2324
|
+
*/
|
|
2325
|
+
export function selectKnowledgeSearchResults(
|
|
2326
|
+
input: KnowledgeSearchSelectionInput,
|
|
2327
|
+
): KnowledgeSearchResponse {
|
|
2328
|
+
const requestedLimit = Math.min(
|
|
2329
|
+
Math.max(Math.trunc(input.requestedLimit), 1),
|
|
2330
|
+
KNOWLEDGE_SEARCH_MAX_RESULTS,
|
|
2331
|
+
);
|
|
2332
|
+
const nowMs = (input.now ?? new Date()).getTime();
|
|
2333
|
+
let belowRelevanceFloor = Math.min(
|
|
2334
|
+
Math.max(0, Math.trunc(input.alreadyBelowRelevanceFloor ?? 0)),
|
|
2335
|
+
KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS,
|
|
2336
|
+
);
|
|
2337
|
+
const relevant: KnowledgeSearchResult[] = [];
|
|
2338
|
+
for (const candidate of input.candidates) {
|
|
2339
|
+
const relevanceSignals: Array<"vector" | "keyword"> = [];
|
|
2340
|
+
if (
|
|
2341
|
+
candidate.vectorScore !== null &&
|
|
2342
|
+
candidate.vectorScore >= KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE
|
|
2343
|
+
) {
|
|
2344
|
+
relevanceSignals.push("vector");
|
|
2345
|
+
}
|
|
2346
|
+
if (
|
|
2347
|
+
candidate.keywordScore !== null &&
|
|
2348
|
+
candidate.keywordScore >= KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE
|
|
2349
|
+
) {
|
|
2350
|
+
relevanceSignals.push("keyword");
|
|
2351
|
+
}
|
|
2352
|
+
if (relevanceSignals.length === 0) {
|
|
2353
|
+
belowRelevanceFloor = Math.min(belowRelevanceFloor + 1, KNOWLEDGE_SEARCH_MAX_FLOOR_OMISSIONS);
|
|
2354
|
+
continue;
|
|
2355
|
+
}
|
|
2356
|
+
const freshness = knowledgeFreshness(candidate.record.quality.freshnessAt, nowMs);
|
|
2357
|
+
const qualityAdjustment = freshness === "current" ? 0.02 : freshness === "aging" ? 0.01 : 0;
|
|
2358
|
+
relevant.push({
|
|
2359
|
+
record: candidate.record,
|
|
2360
|
+
retrieval: {
|
|
2361
|
+
score: roundScore(Math.min(1, candidate.semanticScore + qualityAdjustment)),
|
|
2362
|
+
semanticScore: roundScore(candidate.semanticScore),
|
|
2363
|
+
matchType: candidate.matchType,
|
|
2364
|
+
vectorScore: candidate.vectorScore === null ? null : roundScore(candidate.vectorScore),
|
|
2365
|
+
keywordScore: candidate.keywordScore === null ? null : roundScore(candidate.keywordScore),
|
|
2366
|
+
relevanceSignals,
|
|
2367
|
+
freshness,
|
|
2368
|
+
qualityAdjustment,
|
|
2369
|
+
duplicateCount: 0,
|
|
2370
|
+
},
|
|
2371
|
+
});
|
|
2372
|
+
}
|
|
2373
|
+
relevant.sort(compareKnowledgeSearchResults);
|
|
2374
|
+
|
|
2375
|
+
const deduped: KnowledgeSearchResult[] = [];
|
|
2376
|
+
const byContent = new Map<string, number>();
|
|
2377
|
+
let asDuplicate = 0;
|
|
2378
|
+
for (const result of relevant) {
|
|
2379
|
+
const key = knowledgeTextualContentKey(result.record);
|
|
2380
|
+
const retainedIndex = byContent.get(key);
|
|
2381
|
+
if (retainedIndex === undefined) {
|
|
2382
|
+
byContent.set(key, deduped.length);
|
|
2383
|
+
deduped.push(result);
|
|
2384
|
+
continue;
|
|
2385
|
+
}
|
|
2386
|
+
asDuplicate += 1;
|
|
2387
|
+
const retained = deduped[retainedIndex]!;
|
|
2388
|
+
retained.retrieval.duplicateCount += 1;
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
const forLimit = Math.max(0, deduped.length - requestedLimit);
|
|
2392
|
+
const bounded = deduped.slice(0, requestedLimit);
|
|
2393
|
+
let forResponseBudget = 0;
|
|
2394
|
+
let response = knowledgeSearchResponse({
|
|
2395
|
+
results: bounded,
|
|
2396
|
+
rankedCandidateCount: input.rankedCandidateCount,
|
|
2397
|
+
recheckedCandidateCount: input.candidates.length,
|
|
2398
|
+
belowRelevanceFloor,
|
|
2399
|
+
asDuplicate,
|
|
2400
|
+
forLimit,
|
|
2401
|
+
forResponseBudget,
|
|
2402
|
+
});
|
|
2403
|
+
while (knowledgeResponseBytes(response) > KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES) {
|
|
2404
|
+
if (bounded.length === 0) {
|
|
2405
|
+
throw new Error("knowledge search selection facts exceed the response budget");
|
|
2406
|
+
}
|
|
2407
|
+
bounded.pop();
|
|
2408
|
+
forResponseBudget += 1;
|
|
2409
|
+
response = knowledgeSearchResponse({
|
|
2410
|
+
results: bounded,
|
|
2411
|
+
rankedCandidateCount: input.rankedCandidateCount,
|
|
2412
|
+
recheckedCandidateCount: input.candidates.length,
|
|
2413
|
+
belowRelevanceFloor,
|
|
2414
|
+
asDuplicate,
|
|
2415
|
+
forLimit,
|
|
2416
|
+
forResponseBudget,
|
|
2417
|
+
});
|
|
2418
|
+
}
|
|
2419
|
+
return response;
|
|
2420
|
+
}
|
|
2421
|
+
|
|
2422
|
+
function compareKnowledgeSearchResults(
|
|
2423
|
+
left: KnowledgeSearchResult,
|
|
2424
|
+
right: KnowledgeSearchResult,
|
|
2425
|
+
): number {
|
|
2426
|
+
return (
|
|
2427
|
+
right.retrieval.score - left.retrieval.score ||
|
|
2428
|
+
right.retrieval.semanticScore - left.retrieval.semanticScore ||
|
|
2429
|
+
(right.retrieval.vectorScore ?? 0) - (left.retrieval.vectorScore ?? 0) ||
|
|
2430
|
+
(right.retrieval.keywordScore ?? 0) - (left.retrieval.keywordScore ?? 0) ||
|
|
2431
|
+
(left.record.id === right.record.id ? 0 : left.record.id < right.record.id ? -1 : 1)
|
|
2432
|
+
);
|
|
2433
|
+
}
|
|
2434
|
+
|
|
2435
|
+
function knowledgeFreshness(value: string, nowMs: number): "current" | "aging" | "stale" {
|
|
2436
|
+
const freshnessMs = Date.parse(value);
|
|
2437
|
+
if (!Number.isFinite(freshnessMs)) return "stale";
|
|
2438
|
+
const ageDays = Math.max(0, (nowMs - freshnessMs) / 86_400_000);
|
|
2439
|
+
if (ageDays <= 90) return "current";
|
|
2440
|
+
return ageDays <= 365 ? "aging" : "stale";
|
|
2441
|
+
}
|
|
2442
|
+
|
|
2443
|
+
function knowledgeTextualContentKey(record: KnowledgeRecord): string {
|
|
2444
|
+
return createHash("sha256")
|
|
2445
|
+
.update("opengeni:knowledge-search-content:v1\0")
|
|
2446
|
+
.update(record.title)
|
|
2447
|
+
.update("\0")
|
|
2448
|
+
.update(
|
|
2449
|
+
JSON.stringify({
|
|
2450
|
+
body: record.content.body,
|
|
2451
|
+
summary: record.content.summary,
|
|
2452
|
+
topics: record.content.topics,
|
|
2453
|
+
}),
|
|
2454
|
+
)
|
|
2455
|
+
.digest("hex");
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
function knowledgeSearchResponse(input: {
|
|
2459
|
+
results: KnowledgeSearchResult[];
|
|
2460
|
+
rankedCandidateCount: number;
|
|
2461
|
+
recheckedCandidateCount: number;
|
|
2462
|
+
belowRelevanceFloor: number;
|
|
2463
|
+
asDuplicate: number;
|
|
2464
|
+
forLimit: number;
|
|
2465
|
+
forResponseBudget: number;
|
|
2466
|
+
}): KnowledgeSearchResponse {
|
|
2467
|
+
const selection = {
|
|
2468
|
+
relevanceFloor: {
|
|
2469
|
+
policy: "any_signal" as const,
|
|
2470
|
+
vectorScore: KNOWLEDGE_SEARCH_MIN_VECTOR_SCORE as 0.52,
|
|
2471
|
+
keywordScore: KNOWLEDGE_SEARCH_MIN_KEYWORD_SCORE as 0.01,
|
|
2472
|
+
},
|
|
2473
|
+
dedupe: { policy: "exact_textual_content" as const },
|
|
2474
|
+
candidates: {
|
|
2475
|
+
ranked: input.rankedCandidateCount,
|
|
2476
|
+
rechecked: input.recheckedCandidateCount,
|
|
2477
|
+
omittedOnRecheck: Math.max(0, input.rankedCandidateCount - input.recheckedCandidateCount),
|
|
2478
|
+
},
|
|
2479
|
+
omitted: {
|
|
2480
|
+
belowRelevanceFloor: input.belowRelevanceFloor,
|
|
2481
|
+
asDuplicate: input.asDuplicate,
|
|
2482
|
+
forLimit: input.forLimit,
|
|
2483
|
+
forResponseBudget: input.forResponseBudget,
|
|
2484
|
+
},
|
|
2485
|
+
budget: {
|
|
2486
|
+
maxResults: KNOWLEDGE_SEARCH_MAX_RESULTS as 50,
|
|
2487
|
+
maxResponseBytes: KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES as 65_536,
|
|
2488
|
+
responseBytes: 0,
|
|
2489
|
+
tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
|
|
2490
|
+
estimatedTokens: 0,
|
|
2491
|
+
maxEstimatedTokens: Math.ceil(
|
|
2492
|
+
KNOWLEDGE_SEARCH_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2493
|
+
) as 16_384,
|
|
2494
|
+
},
|
|
2495
|
+
};
|
|
2496
|
+
const response: KnowledgeSearchResponse = {
|
|
2497
|
+
results: [...input.results],
|
|
2498
|
+
selection,
|
|
1875
2499
|
};
|
|
2500
|
+
// The counters themselves contribute a few bytes. Iterate to a fixed point so
|
|
2501
|
+
// responseBytes describes the actual serialized response, not an approximation.
|
|
2502
|
+
for (let index = 0; index < 8; index += 1) {
|
|
2503
|
+
const responseBytes = knowledgeResponseBytes(response);
|
|
2504
|
+
const estimatedTokens = Math.ceil(
|
|
2505
|
+
responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2506
|
+
);
|
|
2507
|
+
if (
|
|
2508
|
+
response.selection.budget.responseBytes === responseBytes &&
|
|
2509
|
+
response.selection.budget.estimatedTokens === estimatedTokens
|
|
2510
|
+
) {
|
|
2511
|
+
break;
|
|
2512
|
+
}
|
|
2513
|
+
response.selection.budget.responseBytes = responseBytes;
|
|
2514
|
+
response.selection.budget.estimatedTokens = estimatedTokens;
|
|
2515
|
+
}
|
|
2516
|
+
return response;
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2519
|
+
function knowledgeResponseBytes(response: KnowledgeSearchResponse): number {
|
|
2520
|
+
return Buffer.byteLength(JSON.stringify(response), "utf8");
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
type KnowledgeBrowseEntry = {
|
|
2524
|
+
record: KnowledgeRecord;
|
|
2525
|
+
cursorAfter: string;
|
|
2526
|
+
};
|
|
2527
|
+
|
|
2528
|
+
/**
|
|
2529
|
+
* Bound a browse page without skipping records. Tail records omitted for the
|
|
2530
|
+
* response budget remain behind the returned cursor. If one complete record is
|
|
2531
|
+
* itself too large, return a deterministic discovery projection; knowledge_get
|
|
2532
|
+
* remains the freshly authorized full-record path.
|
|
2533
|
+
*/
|
|
2534
|
+
export function selectKnowledgeBrowseRecords(input: {
|
|
2535
|
+
entries: KnowledgeBrowseEntry[];
|
|
2536
|
+
hasMoreAfterEntries: boolean;
|
|
2537
|
+
}): KnowledgeBrowseResponse {
|
|
2538
|
+
const entries = input.entries.slice(0, KNOWLEDGE_BROWSE_MAX_LIMIT);
|
|
2539
|
+
const selected = entries.map((entry) => entry.record);
|
|
2540
|
+
let omittedForResponseBudget = 0;
|
|
2541
|
+
let compactedRecordCount = 0;
|
|
2542
|
+
let response = knowledgeBrowseResponse({
|
|
2543
|
+
records: selected,
|
|
2544
|
+
nextCursor: input.hasMoreAfterEntries ? (entries.at(-1)?.cursorAfter ?? null) : null,
|
|
2545
|
+
hasMore: input.hasMoreAfterEntries,
|
|
2546
|
+
omittedForResponseBudget,
|
|
2547
|
+
compactedRecordCount,
|
|
2548
|
+
});
|
|
2549
|
+
while (
|
|
2550
|
+
selected.length > 1 &&
|
|
2551
|
+
knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
|
|
2552
|
+
) {
|
|
2553
|
+
selected.pop();
|
|
2554
|
+
omittedForResponseBudget += 1;
|
|
2555
|
+
response = knowledgeBrowseResponse({
|
|
2556
|
+
records: selected,
|
|
2557
|
+
nextCursor: entries[selected.length - 1]?.cursorAfter ?? null,
|
|
2558
|
+
hasMore: true,
|
|
2559
|
+
omittedForResponseBudget,
|
|
2560
|
+
compactedRecordCount,
|
|
2561
|
+
});
|
|
2562
|
+
}
|
|
2563
|
+
if (
|
|
2564
|
+
selected.length === 1 &&
|
|
2565
|
+
knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES
|
|
2566
|
+
) {
|
|
2567
|
+
selected[0] = compactKnowledgeBrowseRecord(selected[0]!);
|
|
2568
|
+
compactedRecordCount = 1;
|
|
2569
|
+
response = knowledgeBrowseResponse({
|
|
2570
|
+
records: selected,
|
|
2571
|
+
nextCursor:
|
|
2572
|
+
omittedForResponseBudget > 0 || input.hasMoreAfterEntries
|
|
2573
|
+
? (entries[0]?.cursorAfter ?? null)
|
|
2574
|
+
: null,
|
|
2575
|
+
hasMore: omittedForResponseBudget > 0 || input.hasMoreAfterEntries,
|
|
2576
|
+
omittedForResponseBudget,
|
|
2577
|
+
compactedRecordCount,
|
|
2578
|
+
});
|
|
2579
|
+
}
|
|
2580
|
+
if (knowledgeBrowseResponseBytes(response) > KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES) {
|
|
2581
|
+
throw new Error("knowledge browse discovery projection exceeds the response budget");
|
|
2582
|
+
}
|
|
2583
|
+
return response;
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
function compactKnowledgeBrowseRecord(record: KnowledgeRecord): KnowledgeRecord {
|
|
2587
|
+
const fields = new Set<KnowledgeRecord["projection"]["fields"][number]>([
|
|
2588
|
+
...record.projection.fields,
|
|
2589
|
+
"content.body",
|
|
2590
|
+
"content.summary",
|
|
2591
|
+
"content.topics",
|
|
2592
|
+
"content.metadata",
|
|
2593
|
+
"provenance.source.uri",
|
|
2594
|
+
"provenance.source.externalId",
|
|
2595
|
+
"provenance.source.title",
|
|
2596
|
+
"provenance.source.author",
|
|
2597
|
+
"provenance.source.version",
|
|
2598
|
+
"provenance.citation",
|
|
2599
|
+
]);
|
|
2600
|
+
return {
|
|
2601
|
+
...record,
|
|
2602
|
+
content: {
|
|
2603
|
+
format: "markdown",
|
|
2604
|
+
body: null,
|
|
2605
|
+
summary: null,
|
|
2606
|
+
topics: [],
|
|
2607
|
+
metadata: {},
|
|
2608
|
+
},
|
|
2609
|
+
provenance: {
|
|
2610
|
+
...record.provenance,
|
|
2611
|
+
source: {
|
|
2612
|
+
...record.provenance.source,
|
|
2613
|
+
uri: null,
|
|
2614
|
+
externalId: null,
|
|
2615
|
+
title: null,
|
|
2616
|
+
author: null,
|
|
2617
|
+
version: null,
|
|
2618
|
+
},
|
|
2619
|
+
citation: null,
|
|
2620
|
+
},
|
|
2621
|
+
links: record.links.filter((link) => link.target.kind === "knowledge"),
|
|
2622
|
+
projection: {
|
|
2623
|
+
truncated: true,
|
|
2624
|
+
fields: [...fields].sort(),
|
|
2625
|
+
},
|
|
2626
|
+
};
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
function knowledgeBrowseResponse(input: {
|
|
2630
|
+
records: KnowledgeRecord[];
|
|
2631
|
+
nextCursor: string | null;
|
|
2632
|
+
hasMore: boolean;
|
|
2633
|
+
omittedForResponseBudget: number;
|
|
2634
|
+
compactedRecordCount: number;
|
|
2635
|
+
}): KnowledgeBrowseResponse {
|
|
2636
|
+
const response: KnowledgeBrowseResponse = {
|
|
2637
|
+
records: [...input.records],
|
|
2638
|
+
nextCursor: input.nextCursor,
|
|
2639
|
+
hasMore: input.hasMore,
|
|
2640
|
+
selection: {
|
|
2641
|
+
omitted: { forResponseBudget: input.omittedForResponseBudget },
|
|
2642
|
+
compactedRecordCount: input.compactedRecordCount,
|
|
2643
|
+
budget: {
|
|
2644
|
+
maxResults: KNOWLEDGE_BROWSE_MAX_LIMIT as 50,
|
|
2645
|
+
maxResponseBytes: KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES as 65_536,
|
|
2646
|
+
responseBytes: 0,
|
|
2647
|
+
tokenEstimateBytesPerToken: KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN as 4,
|
|
2648
|
+
estimatedTokens: 0,
|
|
2649
|
+
maxEstimatedTokens: Math.ceil(
|
|
2650
|
+
KNOWLEDGE_BROWSE_MAX_RESPONSE_BYTES / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2651
|
+
) as 16_384,
|
|
2652
|
+
},
|
|
2653
|
+
},
|
|
2654
|
+
};
|
|
2655
|
+
for (let index = 0; index < 8; index += 1) {
|
|
2656
|
+
const responseBytes = knowledgeBrowseResponseBytes(response);
|
|
2657
|
+
const estimatedTokens = Math.ceil(
|
|
2658
|
+
responseBytes / KNOWLEDGE_SEARCH_TOKEN_ESTIMATE_BYTES_PER_TOKEN,
|
|
2659
|
+
);
|
|
2660
|
+
if (
|
|
2661
|
+
response.selection.budget.responseBytes === responseBytes &&
|
|
2662
|
+
response.selection.budget.estimatedTokens === estimatedTokens
|
|
2663
|
+
) {
|
|
2664
|
+
break;
|
|
2665
|
+
}
|
|
2666
|
+
response.selection.budget.responseBytes = responseBytes;
|
|
2667
|
+
response.selection.budget.estimatedTokens = estimatedTokens;
|
|
2668
|
+
}
|
|
2669
|
+
return response;
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
function knowledgeBrowseResponseBytes(response: KnowledgeBrowseResponse): number {
|
|
2673
|
+
return Buffer.byteLength(JSON.stringify(response), "utf8");
|
|
1876
2674
|
}
|
|
1877
2675
|
|
|
1878
2676
|
/** Fetch one stable Knowledge record with a fresh authorization check. */
|
|
@@ -1882,12 +2680,20 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1882
2680
|
accountId: string;
|
|
1883
2681
|
workspaceId: string;
|
|
1884
2682
|
initiatingSubjectId: string;
|
|
2683
|
+
surface?: "human" | "agent" | undefined;
|
|
1885
2684
|
id: string;
|
|
2685
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
1886
2686
|
},
|
|
1887
2687
|
): Promise<KnowledgeRecord | null> {
|
|
1888
2688
|
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1889
2689
|
const target = parseKnowledgeRecordId(input.id);
|
|
1890
|
-
const access
|
|
2690
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2691
|
+
accountId: input.accountId,
|
|
2692
|
+
workspaceId: input.workspaceId,
|
|
2693
|
+
initiatingSubjectId,
|
|
2694
|
+
surface: input.surface ?? "agent",
|
|
2695
|
+
agentAuthority: input.agentAuthority,
|
|
2696
|
+
});
|
|
1891
2697
|
return await withDocumentAccountRls(
|
|
1892
2698
|
db,
|
|
1893
2699
|
input.accountId,
|
|
@@ -1895,9 +2701,21 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1895
2701
|
access,
|
|
1896
2702
|
async (scopedDb) => {
|
|
1897
2703
|
if (target.kind === "document") {
|
|
1898
|
-
const [
|
|
1899
|
-
.select(
|
|
2704
|
+
const [row] = await scopedDb
|
|
2705
|
+
.select({
|
|
2706
|
+
document: schema.documents,
|
|
2707
|
+
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2708
|
+
firstChunkId: schema.documentChunks.id,
|
|
2709
|
+
})
|
|
1900
2710
|
.from(schema.documents)
|
|
2711
|
+
.leftJoin(
|
|
2712
|
+
schema.documentChunks,
|
|
2713
|
+
and(
|
|
2714
|
+
eq(schema.documentChunks.accountId, schema.documents.accountId),
|
|
2715
|
+
eq(schema.documentChunks.documentId, schema.documents.id),
|
|
2716
|
+
eq(schema.documentChunks.chunkIndex, 0),
|
|
2717
|
+
),
|
|
2718
|
+
)
|
|
1901
2719
|
.where(
|
|
1902
2720
|
and(
|
|
1903
2721
|
eq(schema.documents.accountId, input.accountId),
|
|
@@ -1907,10 +2725,16 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1907
2725
|
),
|
|
1908
2726
|
)
|
|
1909
2727
|
.limit(1);
|
|
1910
|
-
return
|
|
2728
|
+
return row ? knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId) : null;
|
|
1911
2729
|
}
|
|
1912
2730
|
const [row] = await scopedDb
|
|
1913
|
-
.select({
|
|
2731
|
+
.select({
|
|
2732
|
+
chunk: schema.documentChunks,
|
|
2733
|
+
document: schema.documents,
|
|
2734
|
+
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2735
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
2736
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
2737
|
+
})
|
|
1914
2738
|
.from(schema.documentChunks)
|
|
1915
2739
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1916
2740
|
.where(
|
|
@@ -1923,7 +2747,12 @@ export async function getEffectiveKnowledgeRecord(
|
|
|
1923
2747
|
),
|
|
1924
2748
|
)
|
|
1925
2749
|
.limit(1);
|
|
1926
|
-
return row
|
|
2750
|
+
return row
|
|
2751
|
+
? knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
2752
|
+
previousChunkId: row.previousChunkId,
|
|
2753
|
+
nextChunkId: row.nextChunkId,
|
|
2754
|
+
})
|
|
2755
|
+
: null;
|
|
1927
2756
|
},
|
|
1928
2757
|
);
|
|
1929
2758
|
}
|
|
@@ -1954,7 +2783,7 @@ export async function browseEffectiveKnowledge(
|
|
|
1954
2783
|
if (parent && (topic || sourceKinds.length > 0)) {
|
|
1955
2784
|
throw new Error("knowledge browse document contents do not accept topic/source filters");
|
|
1956
2785
|
}
|
|
1957
|
-
const cursorScope = {
|
|
2786
|
+
const cursorScope: KnowledgeBrowseCursorScope = {
|
|
1958
2787
|
accountId: input.accountId,
|
|
1959
2788
|
workspaceId: input.workspaceId,
|
|
1960
2789
|
initiatingSubjectId,
|
|
@@ -1962,11 +2791,15 @@ export async function browseEffectiveKnowledge(
|
|
|
1962
2791
|
topic,
|
|
1963
2792
|
sourceKinds,
|
|
1964
2793
|
};
|
|
1965
|
-
const
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
2794
|
+
const topLevelAfter =
|
|
2795
|
+
!parent && input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
|
|
2796
|
+
const access = await resolveEffectiveDocumentAccess(db, {
|
|
2797
|
+
accountId: input.accountId,
|
|
2798
|
+
workspaceId: input.workspaceId,
|
|
2799
|
+
initiatingSubjectId,
|
|
2800
|
+
surface: input.surface ?? "agent",
|
|
2801
|
+
agentAuthority: input.agentAuthority,
|
|
2802
|
+
});
|
|
1970
2803
|
return await withDocumentAccountRls(
|
|
1971
2804
|
db,
|
|
1972
2805
|
input.accountId,
|
|
@@ -1975,7 +2808,7 @@ export async function browseEffectiveKnowledge(
|
|
|
1975
2808
|
async (scopedDb) => {
|
|
1976
2809
|
if (parent) {
|
|
1977
2810
|
const [authorizedParent] = await scopedDb
|
|
1978
|
-
.select({ id: schema.documents.id })
|
|
2811
|
+
.select({ id: schema.documents.id, indexSequence: schema.documents.indexSequence })
|
|
1979
2812
|
.from(schema.documents)
|
|
1980
2813
|
.where(
|
|
1981
2814
|
and(
|
|
@@ -1986,9 +2819,30 @@ export async function browseEffectiveKnowledge(
|
|
|
1986
2819
|
),
|
|
1987
2820
|
)
|
|
1988
2821
|
.limit(1);
|
|
1989
|
-
if (!authorizedParent)
|
|
2822
|
+
if (!authorizedParent) {
|
|
2823
|
+
return selectKnowledgeBrowseRecords({ entries: [], hasMoreAfterEntries: false });
|
|
2824
|
+
}
|
|
2825
|
+
if (authorizedParent.indexSequence === null) {
|
|
2826
|
+
throw new Error("ready knowledge document is missing its index revision");
|
|
2827
|
+
}
|
|
2828
|
+
const parentCursorScope: KnowledgeBrowseCursorScope = {
|
|
2829
|
+
...cursorScope,
|
|
2830
|
+
parentRevision: authorizedParent.indexSequence.toString(),
|
|
2831
|
+
};
|
|
2832
|
+
const after = input.cursor
|
|
2833
|
+
? decodeKnowledgeBrowseCursor(input.cursor, parentCursorScope)
|
|
2834
|
+
: 0n;
|
|
2835
|
+
if (after > 2_147_483_648n) {
|
|
2836
|
+
throw new Error("invalid knowledge browse cursor");
|
|
2837
|
+
}
|
|
1990
2838
|
const rows = await scopedDb
|
|
1991
|
-
.select({
|
|
2839
|
+
.select({
|
|
2840
|
+
chunk: schema.documentChunks,
|
|
2841
|
+
document: schema.documents,
|
|
2842
|
+
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2843
|
+
previousChunkId: knowledgePreviousChunkIdProjection(),
|
|
2844
|
+
nextChunkId: knowledgeNextChunkIdProjection(),
|
|
2845
|
+
})
|
|
1992
2846
|
.from(schema.documentChunks)
|
|
1993
2847
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1994
2848
|
.where(
|
|
@@ -1997,6 +2851,7 @@ export async function browseEffectiveKnowledge(
|
|
|
1997
2851
|
eq(schema.documentChunks.documentId, parent.id),
|
|
1998
2852
|
gt(schema.documentChunks.chunkIndex, Number(after) - 1),
|
|
1999
2853
|
eq(schema.documents.status, "ready"),
|
|
2854
|
+
eq(schema.documents.indexSequence, authorizedParent.indexSequence),
|
|
2000
2855
|
...documentAccessConditions(input.workspaceId, access),
|
|
2001
2856
|
),
|
|
2002
2857
|
)
|
|
@@ -2004,44 +2859,63 @@ export async function browseEffectiveKnowledge(
|
|
|
2004
2859
|
.limit(limit + 1);
|
|
2005
2860
|
const hasMore = rows.length > limit;
|
|
2006
2861
|
const page = rows.slice(0, limit);
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2862
|
+
return selectKnowledgeBrowseRecords({
|
|
2863
|
+
entries: page.map((row) => ({
|
|
2864
|
+
record: knowledgeChunkRecord(row.document, row.chunk, row.citation, {
|
|
2865
|
+
previousChunkId: row.previousChunkId,
|
|
2866
|
+
nextChunkId: row.nextChunkId,
|
|
2867
|
+
}),
|
|
2868
|
+
cursorAfter: encodeKnowledgeBrowseCursor(
|
|
2869
|
+
parentCursorScope,
|
|
2870
|
+
BigInt(row.chunk.chunkIndex + 1),
|
|
2871
|
+
),
|
|
2872
|
+
})),
|
|
2873
|
+
hasMoreAfterEntries: hasMore,
|
|
2874
|
+
});
|
|
2016
2875
|
}
|
|
2017
2876
|
|
|
2018
2877
|
const conditions: SQL[] = [
|
|
2019
2878
|
eq(schema.documents.accountId, input.accountId),
|
|
2020
2879
|
eq(schema.documents.status, "ready"),
|
|
2021
2880
|
isNotNull(schema.documents.indexSequence),
|
|
2022
|
-
gt(schema.documents.indexSequence,
|
|
2881
|
+
gt(schema.documents.indexSequence, topLevelAfter),
|
|
2023
2882
|
...documentAccessConditions(input.workspaceId, access),
|
|
2024
2883
|
];
|
|
2025
2884
|
if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
|
|
2026
2885
|
if (sourceKinds.length > 0)
|
|
2027
2886
|
conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
|
|
2028
2887
|
const rows = await scopedDb
|
|
2029
|
-
.select(
|
|
2888
|
+
.select({
|
|
2889
|
+
document: schema.documents,
|
|
2890
|
+
citation: googleDriveCitationProjection(input.workspaceId, access),
|
|
2891
|
+
firstChunkId: schema.documentChunks.id,
|
|
2892
|
+
})
|
|
2030
2893
|
.from(schema.documents)
|
|
2894
|
+
.leftJoin(
|
|
2895
|
+
schema.documentChunks,
|
|
2896
|
+
and(
|
|
2897
|
+
eq(schema.documentChunks.accountId, schema.documents.accountId),
|
|
2898
|
+
eq(schema.documentChunks.documentId, schema.documents.id),
|
|
2899
|
+
eq(schema.documentChunks.chunkIndex, 0),
|
|
2900
|
+
),
|
|
2901
|
+
)
|
|
2031
2902
|
.where(and(...conditions))
|
|
2032
2903
|
.orderBy(asc(schema.documents.indexSequence))
|
|
2033
2904
|
.limit(limit + 1);
|
|
2034
2905
|
const hasMore = rows.length > limit;
|
|
2035
2906
|
const page = rows.slice(0, limit);
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
:
|
|
2043
|
-
|
|
2044
|
-
|
|
2907
|
+
return selectKnowledgeBrowseRecords({
|
|
2908
|
+
entries: page.map((row) => {
|
|
2909
|
+
if (row.document.indexSequence === null) {
|
|
2910
|
+
throw new Error("ready knowledge document is missing its index revision");
|
|
2911
|
+
}
|
|
2912
|
+
return {
|
|
2913
|
+
record: knowledgeDocumentRecord(row.document, row.citation, row.firstChunkId),
|
|
2914
|
+
cursorAfter: encodeKnowledgeBrowseCursor(cursorScope, row.document.indexSequence),
|
|
2915
|
+
};
|
|
2916
|
+
}),
|
|
2917
|
+
hasMoreAfterEntries: hasMore,
|
|
2918
|
+
});
|
|
2045
2919
|
},
|
|
2046
2920
|
);
|
|
2047
2921
|
}
|
|
@@ -2051,6 +2925,7 @@ type KnowledgeBrowseCursorScope = {
|
|
|
2051
2925
|
workspaceId: string;
|
|
2052
2926
|
initiatingSubjectId: string;
|
|
2053
2927
|
parentId: string | null;
|
|
2928
|
+
parentRevision?: string | null | undefined;
|
|
2054
2929
|
topic: string | null;
|
|
2055
2930
|
sourceKinds: readonly string[];
|
|
2056
2931
|
};
|
|
@@ -2060,8 +2935,9 @@ export function encodeKnowledgeBrowseCursor(
|
|
|
2060
2935
|
position: bigint,
|
|
2061
2936
|
): string {
|
|
2062
2937
|
if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
|
|
2938
|
+
const version = knowledgeBrowseCursorVersion(scope);
|
|
2063
2939
|
return Buffer.from(
|
|
2064
|
-
JSON.stringify({ v:
|
|
2940
|
+
JSON.stringify({ v: version, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
|
|
2065
2941
|
"utf8",
|
|
2066
2942
|
).toString("base64url");
|
|
2067
2943
|
}
|
|
@@ -2077,9 +2953,10 @@ export function decodeKnowledgeBrowseCursor(
|
|
|
2077
2953
|
const bytes = Buffer.from(value, "base64url");
|
|
2078
2954
|
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
2079
2955
|
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
2956
|
+
const expectedVersion = knowledgeBrowseCursorVersion(scope);
|
|
2080
2957
|
if (
|
|
2081
2958
|
Object.keys(parsed).sort().join(",") !== "q,s,v" ||
|
|
2082
|
-
parsed.v !==
|
|
2959
|
+
parsed.v !== expectedVersion ||
|
|
2083
2960
|
typeof parsed.s !== "string" ||
|
|
2084
2961
|
typeof parsed.q !== "string" ||
|
|
2085
2962
|
!/^(0|[1-9][0-9]*)$/.test(parsed.q)
|
|
@@ -2104,8 +2981,9 @@ export function decodeKnowledgeBrowseCursor(
|
|
|
2104
2981
|
}
|
|
2105
2982
|
|
|
2106
2983
|
function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
|
|
2107
|
-
|
|
2108
|
-
|
|
2984
|
+
const version = knowledgeBrowseCursorVersion(scope);
|
|
2985
|
+
const hash = createHash("sha256")
|
|
2986
|
+
.update(`opengeni:knowledge-browse-cursor:v${version}\0`)
|
|
2109
2987
|
.update(scope.accountId)
|
|
2110
2988
|
.update("\0")
|
|
2111
2989
|
.update(scope.workspaceId)
|
|
@@ -2113,13 +2991,23 @@ function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
|
|
|
2113
2991
|
.update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
|
|
2114
2992
|
.update("\0")
|
|
2115
2993
|
.update(scope.parentId ?? "")
|
|
2116
|
-
.update("\0")
|
|
2994
|
+
.update("\0");
|
|
2995
|
+
if (version === 2) hash.update(scope.parentRevision!).update("\0");
|
|
2996
|
+
return hash
|
|
2117
2997
|
.update(scope.topic ?? "")
|
|
2118
2998
|
.update("\0")
|
|
2119
2999
|
.update([...scope.sourceKinds].sort().join("\0"))
|
|
2120
3000
|
.digest("hex");
|
|
2121
3001
|
}
|
|
2122
3002
|
|
|
3003
|
+
function knowledgeBrowseCursorVersion(scope: KnowledgeBrowseCursorScope): 1 | 2 {
|
|
3004
|
+
if (!scope.parentId) return 1;
|
|
3005
|
+
if (!scope.parentRevision || !/^[1-9][0-9]*$/.test(scope.parentRevision)) {
|
|
3006
|
+
throw new Error("knowledge browse parent cursor requires an exact document revision");
|
|
3007
|
+
}
|
|
3008
|
+
return 2;
|
|
3009
|
+
}
|
|
3010
|
+
|
|
2123
3011
|
function parseKnowledgeRecordId(value: string): {
|
|
2124
3012
|
kind: "document" | "document_chunk";
|
|
2125
3013
|
id: string;
|
|
@@ -2132,7 +3020,11 @@ function parseKnowledgeRecordId(value: string): {
|
|
|
2132
3020
|
return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
|
|
2133
3021
|
}
|
|
2134
3022
|
|
|
2135
|
-
function knowledgeDocumentRecord(
|
|
3023
|
+
function knowledgeDocumentRecord(
|
|
3024
|
+
document: typeof schema.documents.$inferSelect,
|
|
3025
|
+
citation: unknown = null,
|
|
3026
|
+
firstChunkId: string | null = null,
|
|
3027
|
+
): KnowledgeRecord {
|
|
2136
3028
|
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2137
3029
|
const projected = projectKnowledgeRecord({
|
|
2138
3030
|
title: document.title,
|
|
@@ -2151,10 +3043,24 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
|
|
|
2151
3043
|
provenance: {
|
|
2152
3044
|
source: projected.source,
|
|
2153
3045
|
indexedAt: document.indexedAt.toISOString(),
|
|
3046
|
+
citation: parseKnowledgeProviderCitation(citation),
|
|
2154
3047
|
},
|
|
2155
3048
|
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2156
3049
|
quality: knowledgeQuality(document),
|
|
2157
|
-
links:
|
|
3050
|
+
links: [
|
|
3051
|
+
...(firstChunkId
|
|
3052
|
+
? [
|
|
3053
|
+
{
|
|
3054
|
+
relation: "contents" as const,
|
|
3055
|
+
target: {
|
|
3056
|
+
kind: "knowledge" as const,
|
|
3057
|
+
id: `document_chunk:${firstChunkId}` as const,
|
|
3058
|
+
},
|
|
3059
|
+
},
|
|
3060
|
+
]
|
|
3061
|
+
: []),
|
|
3062
|
+
...knowledgeSourceLinks(projected.source.uri),
|
|
3063
|
+
],
|
|
2158
3064
|
projection: projected.projection,
|
|
2159
3065
|
};
|
|
2160
3066
|
}
|
|
@@ -2162,6 +3068,11 @@ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect)
|
|
|
2162
3068
|
function knowledgeChunkRecord(
|
|
2163
3069
|
document: typeof schema.documents.$inferSelect,
|
|
2164
3070
|
chunk: typeof schema.documentChunks.$inferSelect,
|
|
3071
|
+
citation: unknown = null,
|
|
3072
|
+
traversal: {
|
|
3073
|
+
previousChunkId: string | null;
|
|
3074
|
+
nextChunkId: string | null;
|
|
3075
|
+
} = { previousChunkId: null, nextChunkId: null },
|
|
2165
3076
|
): KnowledgeRecord {
|
|
2166
3077
|
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2167
3078
|
const projected = projectKnowledgeRecord({
|
|
@@ -2181,11 +3092,37 @@ function knowledgeChunkRecord(
|
|
|
2181
3092
|
provenance: {
|
|
2182
3093
|
source: projected.source,
|
|
2183
3094
|
indexedAt: document.indexedAt.toISOString(),
|
|
3095
|
+
citation: parseKnowledgeProviderCitation(citation),
|
|
2184
3096
|
},
|
|
2185
3097
|
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2186
3098
|
quality: knowledgeQuality(document),
|
|
2187
3099
|
links: [
|
|
2188
|
-
{
|
|
3100
|
+
{
|
|
3101
|
+
relation: "parent",
|
|
3102
|
+
target: { kind: "knowledge", id: `document:${document.id}` },
|
|
3103
|
+
},
|
|
3104
|
+
...(traversal.previousChunkId
|
|
3105
|
+
? [
|
|
3106
|
+
{
|
|
3107
|
+
relation: "previous" as const,
|
|
3108
|
+
target: {
|
|
3109
|
+
kind: "knowledge" as const,
|
|
3110
|
+
id: `document_chunk:${traversal.previousChunkId}` as const,
|
|
3111
|
+
},
|
|
3112
|
+
},
|
|
3113
|
+
]
|
|
3114
|
+
: []),
|
|
3115
|
+
...(traversal.nextChunkId
|
|
3116
|
+
? [
|
|
3117
|
+
{
|
|
3118
|
+
relation: "next" as const,
|
|
3119
|
+
target: {
|
|
3120
|
+
kind: "knowledge" as const,
|
|
3121
|
+
id: `document_chunk:${traversal.nextChunkId}` as const,
|
|
3122
|
+
},
|
|
3123
|
+
},
|
|
3124
|
+
]
|
|
3125
|
+
: []),
|
|
2189
3126
|
...knowledgeSourceLinks(projected.source.uri),
|
|
2190
3127
|
],
|
|
2191
3128
|
projection: projected.projection,
|
|
@@ -2221,6 +3158,35 @@ function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"
|
|
|
2221
3158
|
return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
|
|
2222
3159
|
}
|
|
2223
3160
|
|
|
3161
|
+
/**
|
|
3162
|
+
* These structural targets are selected inside the same authorization-scoped
|
|
3163
|
+
* transaction as their owning record. Only opaque ids are projected; titles,
|
|
3164
|
+
* source fields, and content require a subsequent freshly authorized get.
|
|
3165
|
+
*/
|
|
3166
|
+
function knowledgePreviousChunkIdProjection(): SQL<string | null> {
|
|
3167
|
+
return sql<string | null>`(
|
|
3168
|
+
select knowledge_previous_chunk.id
|
|
3169
|
+
from document_chunks knowledge_previous_chunk
|
|
3170
|
+
where knowledge_previous_chunk.account_id = ${schema.documentChunks.accountId}
|
|
3171
|
+
and knowledge_previous_chunk.document_id = ${schema.documentChunks.documentId}
|
|
3172
|
+
and knowledge_previous_chunk.chunk_index < ${schema.documentChunks.chunkIndex}
|
|
3173
|
+
order by knowledge_previous_chunk.chunk_index desc
|
|
3174
|
+
limit 1
|
|
3175
|
+
)`;
|
|
3176
|
+
}
|
|
3177
|
+
|
|
3178
|
+
function knowledgeNextChunkIdProjection(): SQL<string | null> {
|
|
3179
|
+
return sql<string | null>`(
|
|
3180
|
+
select knowledge_next_chunk.id
|
|
3181
|
+
from document_chunks knowledge_next_chunk
|
|
3182
|
+
where knowledge_next_chunk.account_id = ${schema.documentChunks.accountId}
|
|
3183
|
+
and knowledge_next_chunk.document_id = ${schema.documentChunks.documentId}
|
|
3184
|
+
and knowledge_next_chunk.chunk_index > ${schema.documentChunks.chunkIndex}
|
|
3185
|
+
order by knowledge_next_chunk.chunk_index asc
|
|
3186
|
+
limit 1
|
|
3187
|
+
)`;
|
|
3188
|
+
}
|
|
3189
|
+
|
|
2224
3190
|
async function vectorSearchDocuments(
|
|
2225
3191
|
db: Database,
|
|
2226
3192
|
input: DocumentSearchInput,
|
|
@@ -2259,12 +3225,13 @@ async function vectorSearchDocuments(
|
|
|
2259
3225
|
authorityKind: schema.documents.authorityKind,
|
|
2260
3226
|
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
2261
3227
|
authoritySubjectId: schema.documents.authoritySubjectId,
|
|
3228
|
+
citation: googleDriveCitationProjection(input.workspaceId, input.access),
|
|
2262
3229
|
distance,
|
|
2263
3230
|
})
|
|
2264
3231
|
.from(schema.documentChunks)
|
|
2265
3232
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
2266
3233
|
.where(and(...documentSearchConditions(input, services.embedder.model)))
|
|
2267
|
-
.orderBy(distance)
|
|
3234
|
+
.orderBy(distance, asc(schema.documentChunks.id))
|
|
2268
3235
|
.limit(limit),
|
|
2269
3236
|
);
|
|
2270
3237
|
return rows.map((row) => ({
|
|
@@ -2309,6 +3276,7 @@ async function keywordSearchDocuments(
|
|
|
2309
3276
|
authorityKind: schema.documents.authorityKind,
|
|
2310
3277
|
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
2311
3278
|
authoritySubjectId: schema.documents.authoritySubjectId,
|
|
3279
|
+
citation: googleDriveCitationProjection(input.workspaceId, input.access),
|
|
2312
3280
|
rank,
|
|
2313
3281
|
})
|
|
2314
3282
|
.from(schema.documentChunks)
|
|
@@ -2319,7 +3287,7 @@ async function keywordSearchDocuments(
|
|
|
2319
3287
|
sql`to_tsvector('simple', ${schema.documentChunks.text}) @@ plainto_tsquery('simple', ${input.query})`,
|
|
2320
3288
|
),
|
|
2321
3289
|
)
|
|
2322
|
-
.orderBy(desc(rank))
|
|
3290
|
+
.orderBy(desc(rank), asc(schema.documentChunks.id))
|
|
2323
3291
|
.limit(limit),
|
|
2324
3292
|
);
|
|
2325
3293
|
return rows.map((row) => ({
|
|
@@ -2365,6 +3333,7 @@ export async function getDocumentChunk(
|
|
|
2365
3333
|
authorityKind: schema.documents.authorityKind,
|
|
2366
3334
|
authorityWorkspaceId: schema.documents.authorityWorkspaceId,
|
|
2367
3335
|
authoritySubjectId: schema.documents.authoritySubjectId,
|
|
3336
|
+
citation: googleDriveCitationProjection(workspaceId, access),
|
|
2368
3337
|
})
|
|
2369
3338
|
.from(schema.documentChunks)
|
|
2370
3339
|
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
@@ -2468,6 +3437,33 @@ async function assertDocumentAccountWorkspace(
|
|
|
2468
3437
|
return context;
|
|
2469
3438
|
}
|
|
2470
3439
|
|
|
3440
|
+
export async function resolveEffectiveDocumentAccess(
|
|
3441
|
+
_db: Database,
|
|
3442
|
+
input: {
|
|
3443
|
+
accountId: string;
|
|
3444
|
+
workspaceId: string;
|
|
3445
|
+
initiatingSubjectId: string;
|
|
3446
|
+
surface: "human" | "agent";
|
|
3447
|
+
agentAuthority?: AgentDocumentAuthorityContext | undefined;
|
|
3448
|
+
},
|
|
3449
|
+
): Promise<DocumentAccessFilter> {
|
|
3450
|
+
if (input.surface === "human") {
|
|
3451
|
+
return { viewerSubjectId: input.initiatingSubjectId };
|
|
3452
|
+
}
|
|
3453
|
+
return {
|
|
3454
|
+
agentOnly: true,
|
|
3455
|
+
viewerSubjectId: input.initiatingSubjectId,
|
|
3456
|
+
authorizedPersonalAttempt: input.agentAuthority
|
|
3457
|
+
? {
|
|
3458
|
+
accountId: input.accountId,
|
|
3459
|
+
workspaceId: input.workspaceId,
|
|
3460
|
+
sessionId: input.agentAuthority.sessionId,
|
|
3461
|
+
attemptId: input.agentAuthority.attemptId,
|
|
3462
|
+
}
|
|
3463
|
+
: undefined,
|
|
3464
|
+
};
|
|
3465
|
+
}
|
|
3466
|
+
|
|
2471
3467
|
/**
|
|
2472
3468
|
* Visibility/agent scoping shared by every document read path. Fail-closed:
|
|
2473
3469
|
* with no filter supplied, private documents are invisible.
|
|
@@ -2482,32 +3478,70 @@ function documentAccessConditions(
|
|
|
2482
3478
|
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
2483
3479
|
);
|
|
2484
3480
|
const viewer = cleanString(access?.viewerSubjectId ?? null);
|
|
3481
|
+
const personalAttempt = access?.authorizedPersonalAttempt;
|
|
3482
|
+
const authorizedPersonal = personalAttempt
|
|
3483
|
+
? sql`${schema.documents.id} IN (
|
|
3484
|
+
SELECT resolve_session_attempt_personal_document_reads(
|
|
3485
|
+
${personalAttempt.accountId}::uuid,
|
|
3486
|
+
${personalAttempt.workspaceId}::uuid,
|
|
3487
|
+
${personalAttempt.sessionId}::uuid,
|
|
3488
|
+
${personalAttempt.attemptId}::uuid
|
|
3489
|
+
)
|
|
3490
|
+
)`
|
|
3491
|
+
: sql`false`;
|
|
2485
3492
|
const personal = viewer
|
|
2486
3493
|
? and(
|
|
2487
3494
|
eq(schema.documents.authorityKind, "personal"),
|
|
2488
|
-
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
2489
3495
|
eq(schema.documents.authoritySubjectId, viewer),
|
|
3496
|
+
access?.agentOnly
|
|
3497
|
+
? (or(
|
|
3498
|
+
and(
|
|
3499
|
+
isNull(schema.documents.authorityId),
|
|
3500
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
3501
|
+
),
|
|
3502
|
+
and(isNotNull(schema.documents.authorityId), authorizedPersonal),
|
|
3503
|
+
) ?? authorizedPersonal)
|
|
3504
|
+
: (or(
|
|
3505
|
+
eq(schema.documents.authorityWorkspaceId, workspaceId),
|
|
3506
|
+
isNull(schema.documents.authorityWorkspaceId),
|
|
3507
|
+
) ?? eq(schema.documents.authorityWorkspaceId, workspaceId)),
|
|
2490
3508
|
)
|
|
2491
3509
|
: undefined;
|
|
2492
3510
|
const authority = viewer
|
|
2493
3511
|
? (or(organization, workspace, personal) ?? organization)
|
|
2494
3512
|
: (or(organization, workspace) ?? organization);
|
|
3513
|
+
const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
|
|
3514
|
+
const providerAuthorization = sql`google_drive_file_authorized(
|
|
3515
|
+
${schema.documents.accountId},
|
|
3516
|
+
${workspaceId}::uuid,
|
|
3517
|
+
${viewerSql},
|
|
3518
|
+
${schema.documents.fileId}
|
|
3519
|
+
)`;
|
|
2495
3520
|
if (access?.agentOnly) {
|
|
2496
|
-
return [eq(schema.documents.agentAccess, true), authority];
|
|
3521
|
+
return [eq(schema.documents.agentAccess, true), authority, providerAuthorization];
|
|
2497
3522
|
}
|
|
2498
|
-
return [authority];
|
|
3523
|
+
return [authority, providerAuthorization];
|
|
2499
3524
|
}
|
|
2500
3525
|
|
|
2501
3526
|
function documentMatchesAccess(
|
|
2502
3527
|
document: Pick<
|
|
2503
3528
|
DocumentAccessRecord,
|
|
2504
|
-
|
|
3529
|
+
| "id"
|
|
3530
|
+
| "authorityId"
|
|
3531
|
+
| "authorityKind"
|
|
3532
|
+
| "authorityWorkspaceId"
|
|
3533
|
+
| "authoritySubjectId"
|
|
3534
|
+
| "agentAccess"
|
|
2505
3535
|
>,
|
|
2506
3536
|
workspaceId: string,
|
|
2507
3537
|
access: DocumentAccessFilter | undefined,
|
|
2508
3538
|
): boolean {
|
|
2509
3539
|
if (access?.agentOnly) {
|
|
2510
|
-
return
|
|
3540
|
+
return (
|
|
3541
|
+
document.agentAccess &&
|
|
3542
|
+
(document.authorityKind !== "personal" || document.authorityId === null) &&
|
|
3543
|
+
canViewDocument(document, access.viewerSubjectId, workspaceId)
|
|
3544
|
+
);
|
|
2511
3545
|
}
|
|
2512
3546
|
return canViewDocument(document, access?.viewerSubjectId, workspaceId);
|
|
2513
3547
|
}
|
|
@@ -2525,9 +3559,9 @@ function canonicalDocumentAuthoritySubject(value: unknown): string | undefined {
|
|
|
2525
3559
|
/**
|
|
2526
3560
|
* Whether a single already-fetched document is readable in this workspace.
|
|
2527
3561
|
*
|
|
2528
|
-
* Keep this compatibility predicate as strict as SQL/RLS: personal
|
|
2529
|
-
* remains
|
|
2530
|
-
* authority
|
|
3562
|
+
* Keep this compatibility predicate as strict as SQL/RLS: legacy personal
|
|
3563
|
+
* authority remains origin-workspace anchored while activated personal
|
|
3564
|
+
* authority has a null workspace and follows the exact owner within the org.
|
|
2531
3565
|
*/
|
|
2532
3566
|
export function canViewDocument(
|
|
2533
3567
|
document: Pick<DocumentAccessRecord, "authorityKind" | "authoritySubjectId"> & {
|
|
@@ -2540,21 +3574,31 @@ export function canViewDocument(
|
|
|
2540
3574
|
return document.authorityWorkspaceId === null && document.authoritySubjectId === null;
|
|
2541
3575
|
}
|
|
2542
3576
|
const normalizedWorkspaceId = cleanString(workspaceId ?? null);
|
|
2543
|
-
if (!normalizedWorkspaceId
|
|
3577
|
+
if (!normalizedWorkspaceId) {
|
|
2544
3578
|
return false;
|
|
2545
3579
|
}
|
|
2546
3580
|
if (document.authorityKind === "workspace") {
|
|
2547
|
-
return
|
|
3581
|
+
return (
|
|
3582
|
+
document.authorityWorkspaceId === normalizedWorkspaceId &&
|
|
3583
|
+
document.authoritySubjectId === null
|
|
3584
|
+
);
|
|
2548
3585
|
}
|
|
2549
3586
|
if (document.authorityKind === "personal") {
|
|
2550
3587
|
const authoritySubjectId = canonicalDocumentAuthoritySubject(document.authoritySubjectId);
|
|
2551
3588
|
const viewer = canonicalDocumentAuthoritySubject(viewerSubjectId);
|
|
2552
|
-
return
|
|
3589
|
+
return (
|
|
3590
|
+
!!authoritySubjectId &&
|
|
3591
|
+
authoritySubjectId === viewer &&
|
|
3592
|
+
(document.authorityWorkspaceId === null ||
|
|
3593
|
+
document.authorityWorkspaceId === normalizedWorkspaceId)
|
|
3594
|
+
);
|
|
2553
3595
|
}
|
|
2554
3596
|
return false;
|
|
2555
3597
|
}
|
|
2556
3598
|
|
|
2557
3599
|
type DocumentAccessRecord = {
|
|
3600
|
+
id: string;
|
|
3601
|
+
authorityId: string | null;
|
|
2558
3602
|
authorityKind: string;
|
|
2559
3603
|
authorityWorkspaceId: string | null;
|
|
2560
3604
|
authoritySubjectId: string | null;
|
|
@@ -2606,6 +3650,7 @@ function mapSearchRowBase(row: {
|
|
|
2606
3650
|
authorityKind: string;
|
|
2607
3651
|
authorityWorkspaceId: string | null;
|
|
2608
3652
|
authoritySubjectId: string | null;
|
|
3653
|
+
citation?: unknown;
|
|
2609
3654
|
}): SearchRowBase {
|
|
2610
3655
|
return {
|
|
2611
3656
|
chunkId: row.chunkId,
|
|
@@ -2629,9 +3674,29 @@ function mapSearchRowBase(row: {
|
|
|
2629
3674
|
authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
|
|
2630
3675
|
authorityWorkspaceId: row.authorityWorkspaceId,
|
|
2631
3676
|
authoritySubjectId: row.authoritySubjectId,
|
|
3677
|
+
citation: parseKnowledgeProviderCitation(row.citation),
|
|
2632
3678
|
};
|
|
2633
3679
|
}
|
|
2634
3680
|
|
|
3681
|
+
function googleDriveCitationProjection(
|
|
3682
|
+
workspaceId: string,
|
|
3683
|
+
access: DocumentAccessFilter | undefined,
|
|
3684
|
+
): SQL<unknown> {
|
|
3685
|
+
const viewer = cleanString(access?.viewerSubjectId ?? null);
|
|
3686
|
+
const viewerSql = viewer ? sql`${viewer}` : sql`NULL::text`;
|
|
3687
|
+
return sql`google_drive_document_citation(
|
|
3688
|
+
${schema.documents.accountId},
|
|
3689
|
+
${workspaceId}::uuid,
|
|
3690
|
+
${viewerSql},
|
|
3691
|
+
${schema.documents.id},
|
|
3692
|
+
${schema.documents.fileId}
|
|
3693
|
+
)`;
|
|
3694
|
+
}
|
|
3695
|
+
|
|
3696
|
+
function parseKnowledgeProviderCitation(value: unknown): KnowledgeProviderCitationValue | null {
|
|
3697
|
+
return value === null || value === undefined ? null : KnowledgeProviderCitation.parse(value);
|
|
3698
|
+
}
|
|
3699
|
+
|
|
2635
3700
|
function mergeDocumentSearchRows(
|
|
2636
3701
|
rows: CombinedSearchRow[],
|
|
2637
3702
|
mode: DocumentSearchMode,
|
|
@@ -2670,7 +3735,8 @@ function mergeDocumentSearchRows(
|
|
|
2670
3735
|
right.score - left.score ||
|
|
2671
3736
|
(right.vectorScore ?? 0) - (left.vectorScore ?? 0) ||
|
|
2672
3737
|
(right.keywordScore ?? 0) - (left.keywordScore ?? 0) ||
|
|
2673
|
-
left.chunkIndex - right.chunkIndex
|
|
3738
|
+
left.chunkIndex - right.chunkIndex ||
|
|
3739
|
+
(left.chunkId === right.chunkId ? 0 : left.chunkId < right.chunkId ? -1 : 1),
|
|
2674
3740
|
);
|
|
2675
3741
|
}
|
|
2676
3742
|
|
|
@@ -2763,12 +3829,22 @@ export function deterministicEmbedding(
|
|
|
2763
3829
|
|
|
2764
3830
|
async function requireReadyFile(
|
|
2765
3831
|
db: Database,
|
|
2766
|
-
|
|
2767
|
-
|
|
3832
|
+
input: {
|
|
3833
|
+
accountId: string;
|
|
3834
|
+
workspaceId: string;
|
|
3835
|
+
subjectId: string | null;
|
|
3836
|
+
fileId: string;
|
|
3837
|
+
},
|
|
2768
3838
|
): Promise<FileAsset> {
|
|
2769
|
-
const file = await
|
|
3839
|
+
const [file] = await getFilesForSubject(db, {
|
|
3840
|
+
accountId: input.accountId,
|
|
3841
|
+
workspaceId: input.workspaceId,
|
|
3842
|
+
subjectId: input.subjectId,
|
|
3843
|
+
fileIds: [input.fileId],
|
|
3844
|
+
});
|
|
3845
|
+
if (!file) throw new Error(`File not found: ${input.fileId}`);
|
|
2770
3846
|
if (file.status !== "ready") {
|
|
2771
|
-
throw new Error(`File ${fileId} is ${file.status}`);
|
|
3847
|
+
throw new Error(`File ${input.fileId} is ${file.status}`);
|
|
2772
3848
|
}
|
|
2773
3849
|
return file;
|
|
2774
3850
|
}
|
|
@@ -2916,6 +3992,7 @@ function mapDocument(row: typeof schema.documents.$inferSelect): Document {
|
|
|
2916
3992
|
authorityKind: normalizeDocumentAuthorityKind(row.authorityKind),
|
|
2917
3993
|
authorityWorkspaceId: row.authorityWorkspaceId,
|
|
2918
3994
|
authoritySubjectId: row.authoritySubjectId,
|
|
3995
|
+
authorityId: row.authorityId,
|
|
2919
3996
|
visibility: normalizeDocumentVisibility(row.visibility),
|
|
2920
3997
|
createdBy: row.createdBy,
|
|
2921
3998
|
agentAccess: row.agentAccess,
|