@opengeni/documents 0.5.32 → 0.5.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +44 -2
- package/dist/index.js +471 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-projection.d.ts +14 -0
- package/package.json +5 -5
- package/src/index.ts +428 -0
- package/src/knowledge-projection.ts +214 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { Settings } from "@opengeni/config";
|
|
2
|
-
import type { AddDocumentRequest, CreateDocumentBaseRequest, Document, DocumentAuthorityKind, DocumentBase, DocumentCurationStatus, DocumentSearchMode, DocumentSearchResult, DocumentStatus, DocumentVisibility, FileAsset, KnowledgeSourceKind, ListIndexedDocumentsResponse } from "@opengeni/contracts";
|
|
2
|
+
import type { AddDocumentRequest, CreateDocumentBaseRequest, Document, DocumentAuthorityKind, DocumentBase, DocumentCurationStatus, DocumentSearchMode, DocumentSearchResult, DocumentStatus, DocumentVisibility, FileAsset, KnowledgeBrowseResponse, KnowledgeRecord, KnowledgeSearchResponse, KnowledgeSourceKind, ListIndexedDocumentsResponse } from "@opengeni/contracts";
|
|
3
3
|
import { type Database } from "@opengeni/db";
|
|
4
4
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
5
|
+
export { projectKnowledgeRecord } from "./knowledge-projection.js";
|
|
5
6
|
export declare const DEFAULT_DOCUMENT_PARSER = "liteparse";
|
|
6
7
|
export declare const DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
|
|
7
8
|
export declare const DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
|
|
@@ -143,6 +144,18 @@ export type ListEffectiveIndexedDocumentsInput = {
|
|
|
143
144
|
checkpoint?: string | undefined;
|
|
144
145
|
limit?: number | undefined;
|
|
145
146
|
};
|
|
147
|
+
export type EffectiveKnowledgeBrowseInput = {
|
|
148
|
+
accountId: string;
|
|
149
|
+
workspaceId: string;
|
|
150
|
+
/** Immutable human subject accepted for the logical request/turn. */
|
|
151
|
+
initiatingSubjectId: string;
|
|
152
|
+
/** Omit to browse top-level documents; pass a document record id for chunks. */
|
|
153
|
+
parentId?: string | undefined;
|
|
154
|
+
topic?: string | undefined;
|
|
155
|
+
sourceKinds?: KnowledgeSourceKind[] | undefined;
|
|
156
|
+
cursor?: string | undefined;
|
|
157
|
+
limit?: number | undefined;
|
|
158
|
+
};
|
|
146
159
|
export type DocumentIndexHooks = {
|
|
147
160
|
beforeEmbed?: (input: {
|
|
148
161
|
accountId: string;
|
|
@@ -323,6 +336,36 @@ export declare function searchDocuments(db: Database, input: DocumentSearchInput
|
|
|
323
336
|
* request/tool input can replace it with another user's personal authority.
|
|
324
337
|
*/
|
|
325
338
|
export declare function searchEffectiveDocuments(db: Database, input: EffectiveDocumentSearchInput, services?: Pick<DocumentServices, "embedder">): Promise<DocumentSearchResult[]>;
|
|
339
|
+
/**
|
|
340
|
+
* Agent-facing Knowledge search over the existing Documents evidence store.
|
|
341
|
+
* Search selects candidates only after authority filtering; this second read
|
|
342
|
+
* rechecks every exact chunk before projecting it, so a record revoked between
|
|
343
|
+
* ranking and response construction disappears rather than leaking stale data.
|
|
344
|
+
*/
|
|
345
|
+
export declare function searchEffectiveKnowledge(db: Database, input: EffectiveDocumentSearchInput, services?: Pick<DocumentServices, "embedder">): Promise<KnowledgeSearchResponse>;
|
|
346
|
+
/** Fetch one stable Knowledge record with a fresh authorization check. */
|
|
347
|
+
export declare function getEffectiveKnowledgeRecord(db: Database, input: {
|
|
348
|
+
accountId: string;
|
|
349
|
+
workspaceId: string;
|
|
350
|
+
initiatingSubjectId: string;
|
|
351
|
+
id: string;
|
|
352
|
+
}): Promise<KnowledgeRecord | null>;
|
|
353
|
+
/**
|
|
354
|
+
* Browse top-level authorized documents or the chunks of one authorized
|
|
355
|
+
* document. The cursor is opaque and bound to the caller, workspace, parent,
|
|
356
|
+
* and filters. It can change pagination position but can never widen scope.
|
|
357
|
+
*/
|
|
358
|
+
export declare function browseEffectiveKnowledge(db: Database, input: EffectiveKnowledgeBrowseInput): Promise<KnowledgeBrowseResponse>;
|
|
359
|
+
type KnowledgeBrowseCursorScope = {
|
|
360
|
+
accountId: string;
|
|
361
|
+
workspaceId: string;
|
|
362
|
+
initiatingSubjectId: string;
|
|
363
|
+
parentId: string | null;
|
|
364
|
+
topic: string | null;
|
|
365
|
+
sourceKinds: readonly string[];
|
|
366
|
+
};
|
|
367
|
+
export declare function encodeKnowledgeBrowseCursor(scope: KnowledgeBrowseCursorScope, position: bigint): string;
|
|
368
|
+
export declare function decodeKnowledgeBrowseCursor(value: string, scope: KnowledgeBrowseCursorScope): bigint;
|
|
326
369
|
export declare function getDocumentChunk(db: Database, accountId: string, workspaceId: string, chunkId: string, access?: DocumentAccessFilter): Promise<DocumentSearchResult | null>;
|
|
327
370
|
export declare function resolveDocumentAuthority(input: {
|
|
328
371
|
kind?: DocumentAuthorityKind | undefined;
|
|
@@ -349,4 +392,3 @@ type DocumentAccessRecord = {
|
|
|
349
392
|
export declare function parseDocumentBytes(bytes: Uint8Array, file: FileAsset, parser?: DocumentParser): Promise<ParsedDocument>;
|
|
350
393
|
export declare function chunkText(text: string, maxChars?: number, overlapChars?: number): string[];
|
|
351
394
|
export declare function deterministicEmbedding(text: string, dimensions?: number): number[];
|
|
352
|
-
export {};
|
package/dist/index.js
CHANGED
|
@@ -10,6 +10,183 @@ import {
|
|
|
10
10
|
import * as schema from "@opengeni/db/schema";
|
|
11
11
|
import { createHash } from "crypto";
|
|
12
12
|
import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql } from "drizzle-orm";
|
|
13
|
+
import { KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS } from "@opengeni/contracts";
|
|
14
|
+
|
|
15
|
+
// src/knowledge-projection.ts
|
|
16
|
+
import {
|
|
17
|
+
KNOWLEDGE_BODY_MAX_BYTES,
|
|
18
|
+
KNOWLEDGE_METADATA_MAX_BYTES,
|
|
19
|
+
KNOWLEDGE_METADATA_MAX_DEPTH,
|
|
20
|
+
KNOWLEDGE_METADATA_MAX_ITEMS,
|
|
21
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
22
|
+
KNOWLEDGE_SOURCE_URI_MAX_BYTES,
|
|
23
|
+
KNOWLEDGE_SUMMARY_MAX_BYTES,
|
|
24
|
+
KNOWLEDGE_TITLE_MAX_BYTES,
|
|
25
|
+
KNOWLEDGE_TOPIC_MAX_BYTES,
|
|
26
|
+
KNOWLEDGE_TOPICS_MAX_ITEMS
|
|
27
|
+
} from "@opengeni/contracts";
|
|
28
|
+
var utf8Bytes = (value) => Buffer.byteLength(value, "utf8");
|
|
29
|
+
function truncateUtf8(value, maxBytes) {
|
|
30
|
+
if (utf8Bytes(value) <= maxBytes) return { value, truncated: false };
|
|
31
|
+
let low = 0;
|
|
32
|
+
let high = value.length;
|
|
33
|
+
while (low < high) {
|
|
34
|
+
const middle = Math.ceil((low + high) / 2);
|
|
35
|
+
const end2 = middle > 0 && middle < value.length && /[\uD800-\uDBFF]/u.test(value[middle - 1]) ? middle - 1 : middle;
|
|
36
|
+
if (utf8Bytes(value.slice(0, end2)) <= maxBytes) low = middle;
|
|
37
|
+
else high = middle - 1;
|
|
38
|
+
}
|
|
39
|
+
let end = low;
|
|
40
|
+
if (end > 0 && end < value.length && /[\uD800-\uDBFF]/u.test(value[end - 1])) end -= 1;
|
|
41
|
+
while (end > 0 && utf8Bytes(value.slice(0, end)) > maxBytes) end -= 1;
|
|
42
|
+
return { value: value.slice(0, end), truncated: true };
|
|
43
|
+
}
|
|
44
|
+
function projectNullableString(value, maxBytes, field, fields) {
|
|
45
|
+
if (value === null) return null;
|
|
46
|
+
const projected = truncateUtf8(value, maxBytes);
|
|
47
|
+
if (projected.truncated) fields.add(field);
|
|
48
|
+
return projected.value;
|
|
49
|
+
}
|
|
50
|
+
function projectSourceUri(value, fields) {
|
|
51
|
+
if (value === null || value === "") return null;
|
|
52
|
+
if (utf8Bytes(value) > KNOWLEDGE_SOURCE_URI_MAX_BYTES) {
|
|
53
|
+
fields.add("provenance.source.uri");
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
function projectTopics(value, fields) {
|
|
59
|
+
if (!Array.isArray(value)) {
|
|
60
|
+
if (value !== null && value !== void 0) fields.add("content.topics");
|
|
61
|
+
return [];
|
|
62
|
+
}
|
|
63
|
+
const result = [];
|
|
64
|
+
for (const candidate of value) {
|
|
65
|
+
if (result.length >= KNOWLEDGE_TOPICS_MAX_ITEMS) {
|
|
66
|
+
fields.add("content.topics");
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
if (typeof candidate !== "string") {
|
|
70
|
+
fields.add("content.topics");
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
const projected = truncateUtf8(candidate, KNOWLEDGE_TOPIC_MAX_BYTES);
|
|
74
|
+
if (projected.truncated) fields.add("content.topics");
|
|
75
|
+
result.push(projected.value);
|
|
76
|
+
}
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
function projectMetadata(value) {
|
|
80
|
+
let remainingItems = KNOWLEDGE_METADATA_MAX_ITEMS;
|
|
81
|
+
let truncated = false;
|
|
82
|
+
const visit = (candidate, depth) => {
|
|
83
|
+
if (remainingItems <= 0 || depth > KNOWLEDGE_METADATA_MAX_DEPTH) {
|
|
84
|
+
truncated = true;
|
|
85
|
+
return void 0;
|
|
86
|
+
}
|
|
87
|
+
remainingItems -= 1;
|
|
88
|
+
if (candidate === null || typeof candidate === "boolean" || typeof candidate === "number" && Number.isFinite(candidate)) {
|
|
89
|
+
return candidate;
|
|
90
|
+
}
|
|
91
|
+
if (typeof candidate === "string") {
|
|
92
|
+
const projected = truncateUtf8(candidate, KNOWLEDGE_SOURCE_STRING_MAX_BYTES);
|
|
93
|
+
if (projected.truncated) truncated = true;
|
|
94
|
+
return projected.value;
|
|
95
|
+
}
|
|
96
|
+
if (Array.isArray(candidate)) {
|
|
97
|
+
const result2 = [];
|
|
98
|
+
for (const item of candidate) {
|
|
99
|
+
const projected = visit(item, depth + 1);
|
|
100
|
+
if (projected === void 0) break;
|
|
101
|
+
result2.push(projected);
|
|
102
|
+
}
|
|
103
|
+
return result2;
|
|
104
|
+
}
|
|
105
|
+
if (candidate && typeof candidate === "object") {
|
|
106
|
+
const result2 = {};
|
|
107
|
+
for (const key of Object.keys(candidate).sort()) {
|
|
108
|
+
if (utf8Bytes(key) > KNOWLEDGE_TOPIC_MAX_BYTES) {
|
|
109
|
+
truncated = true;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
const projected = visit(candidate[key], depth + 1);
|
|
113
|
+
if (projected === void 0) break;
|
|
114
|
+
result2[key] = projected;
|
|
115
|
+
}
|
|
116
|
+
return result2;
|
|
117
|
+
}
|
|
118
|
+
truncated = true;
|
|
119
|
+
return void 0;
|
|
120
|
+
};
|
|
121
|
+
const root = visit(value, 0);
|
|
122
|
+
let result = root && typeof root === "object" && !Array.isArray(root) ? root : {};
|
|
123
|
+
if (result !== root) truncated = true;
|
|
124
|
+
const bounded = {};
|
|
125
|
+
for (const key of Object.keys(result).sort()) {
|
|
126
|
+
bounded[key] = result[key];
|
|
127
|
+
if (utf8Bytes(JSON.stringify(bounded)) > KNOWLEDGE_METADATA_MAX_BYTES) {
|
|
128
|
+
delete bounded[key];
|
|
129
|
+
truncated = true;
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
result = bounded;
|
|
134
|
+
return { value: result, truncated };
|
|
135
|
+
}
|
|
136
|
+
function projectKnowledgeRecord(input) {
|
|
137
|
+
const fields = /* @__PURE__ */ new Set();
|
|
138
|
+
const title = truncateUtf8(input.title, KNOWLEDGE_TITLE_MAX_BYTES);
|
|
139
|
+
if (title.truncated) fields.add("title");
|
|
140
|
+
const metadata = projectMetadata(input.metadata);
|
|
141
|
+
if (metadata.truncated) fields.add("content.metadata");
|
|
142
|
+
const source = {
|
|
143
|
+
...input.source,
|
|
144
|
+
uri: projectSourceUri(input.source.uri, fields),
|
|
145
|
+
externalId: projectNullableString(
|
|
146
|
+
input.source.externalId,
|
|
147
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
148
|
+
"provenance.source.externalId",
|
|
149
|
+
fields
|
|
150
|
+
),
|
|
151
|
+
title: projectNullableString(
|
|
152
|
+
input.source.title,
|
|
153
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
154
|
+
"provenance.source.title",
|
|
155
|
+
fields
|
|
156
|
+
),
|
|
157
|
+
author: projectNullableString(
|
|
158
|
+
input.source.author,
|
|
159
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
160
|
+
"provenance.source.author",
|
|
161
|
+
fields
|
|
162
|
+
),
|
|
163
|
+
version: projectNullableString(
|
|
164
|
+
input.source.version,
|
|
165
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
166
|
+
"provenance.source.version",
|
|
167
|
+
fields
|
|
168
|
+
)
|
|
169
|
+
};
|
|
170
|
+
return {
|
|
171
|
+
title: title.value,
|
|
172
|
+
content: {
|
|
173
|
+
format: "markdown",
|
|
174
|
+
body: projectNullableString(input.body, KNOWLEDGE_BODY_MAX_BYTES, "content.body", fields),
|
|
175
|
+
summary: projectNullableString(
|
|
176
|
+
input.summary,
|
|
177
|
+
KNOWLEDGE_SUMMARY_MAX_BYTES,
|
|
178
|
+
"content.summary",
|
|
179
|
+
fields
|
|
180
|
+
),
|
|
181
|
+
topics: projectTopics(input.topics, fields),
|
|
182
|
+
metadata: metadata.value
|
|
183
|
+
},
|
|
184
|
+
source,
|
|
185
|
+
projection: { truncated: fields.size > 0, fields: [...fields].sort() }
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// src/index.ts
|
|
13
190
|
var DEFAULT_DOCUMENT_PARSER = "liteparse";
|
|
14
191
|
var DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
|
|
15
192
|
var DEFAULT_DOCUMENT_EMBEDDING_DIMENSIONS = 3072;
|
|
@@ -1134,6 +1311,293 @@ async function searchEffectiveDocuments(db, input, services = createDocumentServ
|
|
|
1134
1311
|
services
|
|
1135
1312
|
);
|
|
1136
1313
|
}
|
|
1314
|
+
async function searchEffectiveKnowledge(db, input, services = createDocumentServices()) {
|
|
1315
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1316
|
+
const ranked = await searchEffectiveDocuments(
|
|
1317
|
+
db,
|
|
1318
|
+
{ ...input, initiatingSubjectId, surface: "agent" },
|
|
1319
|
+
services
|
|
1320
|
+
);
|
|
1321
|
+
if (ranked.length === 0) return { results: [] };
|
|
1322
|
+
const access = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1323
|
+
const current = await withDocumentAccountRls(
|
|
1324
|
+
db,
|
|
1325
|
+
input.accountId,
|
|
1326
|
+
input.workspaceId,
|
|
1327
|
+
access,
|
|
1328
|
+
async (scopedDb) => await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
|
|
1329
|
+
and(
|
|
1330
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1331
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1332
|
+
inArray(
|
|
1333
|
+
schema.documentChunks.id,
|
|
1334
|
+
ranked.map((result) => result.chunkId)
|
|
1335
|
+
),
|
|
1336
|
+
eq(schema.documents.status, "ready"),
|
|
1337
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1338
|
+
)
|
|
1339
|
+
)
|
|
1340
|
+
);
|
|
1341
|
+
const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
|
|
1342
|
+
return {
|
|
1343
|
+
results: ranked.flatMap((rankedResult) => {
|
|
1344
|
+
const row = currentByChunkId.get(rankedResult.chunkId);
|
|
1345
|
+
if (!row) return [];
|
|
1346
|
+
return [
|
|
1347
|
+
{
|
|
1348
|
+
record: knowledgeChunkRecord(row.document, row.chunk),
|
|
1349
|
+
retrieval: {
|
|
1350
|
+
score: rankedResult.score,
|
|
1351
|
+
matchType: rankedResult.matchType,
|
|
1352
|
+
vectorScore: rankedResult.vectorScore,
|
|
1353
|
+
keywordScore: rankedResult.keywordScore
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
];
|
|
1357
|
+
})
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
async function getEffectiveKnowledgeRecord(db, input) {
|
|
1361
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1362
|
+
const target = parseKnowledgeRecordId(input.id);
|
|
1363
|
+
const access = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1364
|
+
return await withDocumentAccountRls(
|
|
1365
|
+
db,
|
|
1366
|
+
input.accountId,
|
|
1367
|
+
input.workspaceId,
|
|
1368
|
+
access,
|
|
1369
|
+
async (scopedDb) => {
|
|
1370
|
+
if (target.kind === "document") {
|
|
1371
|
+
const [document] = await scopedDb.select().from(schema.documents).where(
|
|
1372
|
+
and(
|
|
1373
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1374
|
+
eq(schema.documents.id, target.id),
|
|
1375
|
+
eq(schema.documents.status, "ready"),
|
|
1376
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1377
|
+
)
|
|
1378
|
+
).limit(1);
|
|
1379
|
+
return document ? knowledgeDocumentRecord(document) : null;
|
|
1380
|
+
}
|
|
1381
|
+
const [row] = await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
|
|
1382
|
+
and(
|
|
1383
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1384
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1385
|
+
eq(schema.documentChunks.id, target.id),
|
|
1386
|
+
eq(schema.documents.status, "ready"),
|
|
1387
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1388
|
+
)
|
|
1389
|
+
).limit(1);
|
|
1390
|
+
return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
|
|
1391
|
+
}
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
async function browseEffectiveKnowledge(db, input) {
|
|
1395
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1396
|
+
const limit = input.limit ?? 20;
|
|
1397
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 50) {
|
|
1398
|
+
throw new Error("knowledge browse limit must be between 1 and 50");
|
|
1399
|
+
}
|
|
1400
|
+
const topic = input.topic === void 0 ? null : cleanString(input.topic) ?? null;
|
|
1401
|
+
if (input.topic !== void 0 && (!topic || topic !== input.topic || topic.length > 256)) {
|
|
1402
|
+
throw new Error("knowledge browse topic is invalid");
|
|
1403
|
+
}
|
|
1404
|
+
const sourceKinds = [...new Set(input.sourceKinds ?? [])].sort();
|
|
1405
|
+
const parent = input.parentId ? parseKnowledgeRecordId(input.parentId) : null;
|
|
1406
|
+
if (parent?.kind === "document_chunk") {
|
|
1407
|
+
throw new Error("knowledge browse parent must be a document record");
|
|
1408
|
+
}
|
|
1409
|
+
if (parent && (topic || sourceKinds.length > 0)) {
|
|
1410
|
+
throw new Error("knowledge browse document contents do not accept topic/source filters");
|
|
1411
|
+
}
|
|
1412
|
+
const cursorScope = {
|
|
1413
|
+
accountId: input.accountId,
|
|
1414
|
+
workspaceId: input.workspaceId,
|
|
1415
|
+
initiatingSubjectId,
|
|
1416
|
+
parentId: parent ? `${parent.kind}:${parent.id}` : null,
|
|
1417
|
+
topic,
|
|
1418
|
+
sourceKinds
|
|
1419
|
+
};
|
|
1420
|
+
const after = input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
|
|
1421
|
+
if (parent && after > 2147483648n) {
|
|
1422
|
+
throw new Error("invalid knowledge browse cursor");
|
|
1423
|
+
}
|
|
1424
|
+
const access = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1425
|
+
return await withDocumentAccountRls(
|
|
1426
|
+
db,
|
|
1427
|
+
input.accountId,
|
|
1428
|
+
input.workspaceId,
|
|
1429
|
+
access,
|
|
1430
|
+
async (scopedDb) => {
|
|
1431
|
+
if (parent) {
|
|
1432
|
+
const [authorizedParent] = await scopedDb.select({ id: schema.documents.id }).from(schema.documents).where(
|
|
1433
|
+
and(
|
|
1434
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1435
|
+
eq(schema.documents.id, parent.id),
|
|
1436
|
+
eq(schema.documents.status, "ready"),
|
|
1437
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1438
|
+
)
|
|
1439
|
+
).limit(1);
|
|
1440
|
+
if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
|
|
1441
|
+
const rows2 = await scopedDb.select({ chunk: schema.documentChunks, document: schema.documents }).from(schema.documentChunks).innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id)).where(
|
|
1442
|
+
and(
|
|
1443
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1444
|
+
eq(schema.documentChunks.documentId, parent.id),
|
|
1445
|
+
gt(schema.documentChunks.chunkIndex, Number(after) - 1),
|
|
1446
|
+
eq(schema.documents.status, "ready"),
|
|
1447
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1448
|
+
)
|
|
1449
|
+
).orderBy(asc(schema.documentChunks.chunkIndex)).limit(limit + 1);
|
|
1450
|
+
const hasMore2 = rows2.length > limit;
|
|
1451
|
+
const page2 = rows2.slice(0, limit);
|
|
1452
|
+
const last2 = page2.at(-1)?.chunk.chunkIndex;
|
|
1453
|
+
return {
|
|
1454
|
+
records: page2.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
|
|
1455
|
+
nextCursor: hasMore2 && last2 !== void 0 ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last2 + 1)) : null,
|
|
1456
|
+
hasMore: hasMore2
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
const conditions = [
|
|
1460
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1461
|
+
eq(schema.documents.status, "ready"),
|
|
1462
|
+
isNotNull(schema.documents.indexSequence),
|
|
1463
|
+
gt(schema.documents.indexSequence, after),
|
|
1464
|
+
...documentAccessConditions(input.workspaceId, access)
|
|
1465
|
+
];
|
|
1466
|
+
if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
|
|
1467
|
+
if (sourceKinds.length > 0)
|
|
1468
|
+
conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
|
|
1469
|
+
const rows = await scopedDb.select().from(schema.documents).where(and(...conditions)).orderBy(asc(schema.documents.indexSequence)).limit(limit + 1);
|
|
1470
|
+
const hasMore = rows.length > limit;
|
|
1471
|
+
const page = rows.slice(0, limit);
|
|
1472
|
+
const last = page.at(-1)?.indexSequence;
|
|
1473
|
+
return {
|
|
1474
|
+
records: page.map(knowledgeDocumentRecord),
|
|
1475
|
+
nextCursor: hasMore && last !== void 0 && last !== null ? encodeKnowledgeBrowseCursor(cursorScope, last) : null,
|
|
1476
|
+
hasMore
|
|
1477
|
+
};
|
|
1478
|
+
}
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
function encodeKnowledgeBrowseCursor(scope, position) {
|
|
1482
|
+
if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
|
|
1483
|
+
return Buffer.from(
|
|
1484
|
+
JSON.stringify({ v: 1, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
|
|
1485
|
+
"utf8"
|
|
1486
|
+
).toString("base64url");
|
|
1487
|
+
}
|
|
1488
|
+
function decodeKnowledgeBrowseCursor(value, scope) {
|
|
1489
|
+
try {
|
|
1490
|
+
if (!value || value.length > KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS) {
|
|
1491
|
+
throw new Error("cursor length");
|
|
1492
|
+
}
|
|
1493
|
+
const bytes = Buffer.from(value, "base64url");
|
|
1494
|
+
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
1495
|
+
const parsed = JSON.parse(bytes.toString("utf8"));
|
|
1496
|
+
if (Object.keys(parsed).sort().join(",") !== "q,s,v" || parsed.v !== 1 || typeof parsed.s !== "string" || typeof parsed.q !== "string" || !/^(0|[1-9][0-9]*)$/.test(parsed.q)) {
|
|
1497
|
+
throw new Error("cursor payload");
|
|
1498
|
+
}
|
|
1499
|
+
if (parsed.s !== knowledgeBrowseCursorScope(scope)) {
|
|
1500
|
+
throw new Error("knowledge browse cursor belongs to a different scope");
|
|
1501
|
+
}
|
|
1502
|
+
const position = BigInt(parsed.q);
|
|
1503
|
+
if (position > 9223372036854775807n) throw new Error("cursor position");
|
|
1504
|
+
return position;
|
|
1505
|
+
} catch (error) {
|
|
1506
|
+
if (error instanceof Error && error.message === "knowledge browse cursor belongs to a different scope") {
|
|
1507
|
+
throw error;
|
|
1508
|
+
}
|
|
1509
|
+
throw new Error("invalid knowledge browse cursor", { cause: error });
|
|
1510
|
+
}
|
|
1511
|
+
}
|
|
1512
|
+
function knowledgeBrowseCursorScope(scope) {
|
|
1513
|
+
return createHash("sha256").update("opengeni:knowledge-browse-cursor:v1\0").update(scope.accountId).update("\0").update(scope.workspaceId).update("\0").update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId)).update("\0").update(scope.parentId ?? "").update("\0").update(scope.topic ?? "").update("\0").update([...scope.sourceKinds].sort().join("\0")).digest("hex");
|
|
1514
|
+
}
|
|
1515
|
+
function parseKnowledgeRecordId(value) {
|
|
1516
|
+
const match = /^(document|document_chunk):([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/iu.exec(
|
|
1517
|
+
value
|
|
1518
|
+
);
|
|
1519
|
+
if (!match) throw new Error("invalid knowledge record id");
|
|
1520
|
+
return { kind: match[1], id: match[2].toLowerCase() };
|
|
1521
|
+
}
|
|
1522
|
+
function knowledgeDocumentRecord(document) {
|
|
1523
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
1524
|
+
const projected = projectKnowledgeRecord({
|
|
1525
|
+
title: document.title,
|
|
1526
|
+
body: document.summary,
|
|
1527
|
+
summary: document.summary,
|
|
1528
|
+
topics: document.topics,
|
|
1529
|
+
metadata: { parser: document.parser, chunkCount: document.chunkCount },
|
|
1530
|
+
source: knowledgeSource(document)
|
|
1531
|
+
});
|
|
1532
|
+
return {
|
|
1533
|
+
id: `document:${document.id}`,
|
|
1534
|
+
kind: "document",
|
|
1535
|
+
title: projected.title,
|
|
1536
|
+
content: projected.content,
|
|
1537
|
+
authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
|
|
1538
|
+
provenance: {
|
|
1539
|
+
source: projected.source,
|
|
1540
|
+
indexedAt: document.indexedAt.toISOString()
|
|
1541
|
+
},
|
|
1542
|
+
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
1543
|
+
quality: knowledgeQuality(document),
|
|
1544
|
+
links: knowledgeSourceLinks(projected.source.uri),
|
|
1545
|
+
projection: projected.projection
|
|
1546
|
+
};
|
|
1547
|
+
}
|
|
1548
|
+
function knowledgeChunkRecord(document, chunk) {
|
|
1549
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
1550
|
+
const projected = projectKnowledgeRecord({
|
|
1551
|
+
title: document.title,
|
|
1552
|
+
body: chunk.text,
|
|
1553
|
+
summary: document.summary,
|
|
1554
|
+
topics: document.topics,
|
|
1555
|
+
metadata: { ...chunk.metadata, chunkIndex: chunk.chunkIndex },
|
|
1556
|
+
source: knowledgeSource(document)
|
|
1557
|
+
});
|
|
1558
|
+
return {
|
|
1559
|
+
id: `document_chunk:${chunk.id}`,
|
|
1560
|
+
kind: "document_chunk",
|
|
1561
|
+
title: projected.title,
|
|
1562
|
+
content: projected.content,
|
|
1563
|
+
authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
|
|
1564
|
+
provenance: {
|
|
1565
|
+
source: projected.source,
|
|
1566
|
+
indexedAt: document.indexedAt.toISOString()
|
|
1567
|
+
},
|
|
1568
|
+
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
1569
|
+
quality: knowledgeQuality(document),
|
|
1570
|
+
links: [
|
|
1571
|
+
{ relation: "parent", target: { kind: "knowledge", id: `document:${document.id}` } },
|
|
1572
|
+
...knowledgeSourceLinks(projected.source.uri)
|
|
1573
|
+
],
|
|
1574
|
+
projection: projected.projection
|
|
1575
|
+
};
|
|
1576
|
+
}
|
|
1577
|
+
function knowledgeSource(document) {
|
|
1578
|
+
return {
|
|
1579
|
+
kind: normalizeKnowledgeSourceKind(document.sourceKind),
|
|
1580
|
+
uri: document.sourceUri,
|
|
1581
|
+
externalId: document.sourceExternalId,
|
|
1582
|
+
title: document.sourceTitle,
|
|
1583
|
+
author: document.sourceAuthor,
|
|
1584
|
+
createdAt: document.sourceCreatedAt?.toISOString() ?? null,
|
|
1585
|
+
updatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
|
|
1586
|
+
version: document.sourceVersion
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
function knowledgeQuality(document) {
|
|
1590
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
1591
|
+
return {
|
|
1592
|
+
trust: "sourced",
|
|
1593
|
+
freshnessAt: (document.sourceUpdatedAt ?? document.indexedAt).toISOString(),
|
|
1594
|
+
conflict: "not_evaluated",
|
|
1595
|
+
correction: "current_source_version"
|
|
1596
|
+
};
|
|
1597
|
+
}
|
|
1598
|
+
function knowledgeSourceLinks(sourceUri) {
|
|
1599
|
+
return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
|
|
1600
|
+
}
|
|
1137
1601
|
async function vectorSearchDocuments(db, input, limit, services) {
|
|
1138
1602
|
const queryEmbedding = await services.embedder.embedQuery(input.query);
|
|
1139
1603
|
validateEmbedding(queryEmbedding, services.embedder.dimensions, services.embedder.model);
|
|
@@ -1695,20 +2159,24 @@ export {
|
|
|
1695
2159
|
OpenAIEmbeddingProvider,
|
|
1696
2160
|
RecursiveTextChunker,
|
|
1697
2161
|
addDocumentToBase,
|
|
2162
|
+
browseEffectiveKnowledge,
|
|
1698
2163
|
canViewDocument,
|
|
1699
2164
|
chunkText,
|
|
1700
2165
|
createDocumentBase,
|
|
1701
2166
|
createDocumentServices,
|
|
1702
2167
|
decodeDocumentIndexCheckpoint,
|
|
2168
|
+
decodeKnowledgeBrowseCursor,
|
|
1703
2169
|
deleteDocumentFromBase,
|
|
1704
2170
|
deterministicEmbedding,
|
|
1705
2171
|
documentOpenAIEmbeddingConfig,
|
|
1706
2172
|
encodeDocumentIndexCheckpoint,
|
|
2173
|
+
encodeKnowledgeBrowseCursor,
|
|
1707
2174
|
ensureDefaultBase,
|
|
1708
2175
|
getDocument,
|
|
1709
2176
|
getDocumentBase,
|
|
1710
2177
|
getDocumentChunk,
|
|
1711
2178
|
getDocumentInventory,
|
|
2179
|
+
getEffectiveKnowledgeRecord,
|
|
1712
2180
|
heuristicCuration,
|
|
1713
2181
|
indexDocumentNow,
|
|
1714
2182
|
listDocumentBases,
|
|
@@ -1718,9 +2186,11 @@ export {
|
|
|
1718
2186
|
moveDocumentToBase,
|
|
1719
2187
|
parseCurationOutcome,
|
|
1720
2188
|
parseDocumentBytes,
|
|
2189
|
+
projectKnowledgeRecord,
|
|
1721
2190
|
queueDocumentForReindex,
|
|
1722
2191
|
resolveDocumentAuthority,
|
|
1723
2192
|
searchDocuments,
|
|
1724
|
-
searchEffectiveDocuments
|
|
2193
|
+
searchEffectiveDocuments,
|
|
2194
|
+
searchEffectiveKnowledge
|
|
1725
2195
|
};
|
|
1726
2196
|
//# sourceMappingURL=index.js.map
|