@opengeni/documents 0.5.32 → 0.5.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +44 -2
- package/dist/index.js +471 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-projection.d.ts +14 -0
- package/package.json +5 -5
- package/src/index.ts +428 -0
- package/src/knowledge-projection.ts +214 -0
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type KnowledgeRecord, type KnowledgeSource } from "@opengeni/contracts";
|
|
2
|
+
export type KnowledgeProjectionInput = {
|
|
3
|
+
title: string;
|
|
4
|
+
body: string | null;
|
|
5
|
+
summary: string | null;
|
|
6
|
+
topics: unknown;
|
|
7
|
+
metadata: unknown;
|
|
8
|
+
source: KnowledgeSource;
|
|
9
|
+
};
|
|
10
|
+
export type KnowledgeProjectionResult = Pick<KnowledgeRecord, "title" | "content" | "projection"> & {
|
|
11
|
+
source: KnowledgeSource;
|
|
12
|
+
};
|
|
13
|
+
/** Deterministic, byte-bounded projection at the agent-facing Knowledge envelope. */
|
|
14
|
+
export declare function projectKnowledgeRecord(input: KnowledgeProjectionInput): KnowledgeProjectionResult;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/documents",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.38",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -40,10 +40,10 @@
|
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
42
|
"@llamaindex/liteparse": "^1.5.3",
|
|
43
|
-
"@opengeni/config": "^0.
|
|
44
|
-
"@opengeni/contracts": "^0.
|
|
45
|
-
"@opengeni/db": "^0.
|
|
46
|
-
"@opengeni/storage": "^0.2.
|
|
43
|
+
"@opengeni/config": "^0.16.1",
|
|
44
|
+
"@opengeni/contracts": "^0.50.0",
|
|
45
|
+
"@opengeni/db": "^0.36.0",
|
|
46
|
+
"@opengeni/storage": "^0.2.93",
|
|
47
47
|
"drizzle-orm": "^0.45.2",
|
|
48
48
|
"openai": "6.47.0"
|
|
49
49
|
}
|
package/src/index.ts
CHANGED
|
@@ -13,6 +13,9 @@ import type {
|
|
|
13
13
|
DocumentVisibility,
|
|
14
14
|
FileAsset,
|
|
15
15
|
IndexedDocumentSummary,
|
|
16
|
+
KnowledgeBrowseResponse,
|
|
17
|
+
KnowledgeRecord,
|
|
18
|
+
KnowledgeSearchResponse,
|
|
16
19
|
KnowledgeSourceKind,
|
|
17
20
|
ListIndexedDocumentsResponse,
|
|
18
21
|
} from "@opengeni/contracts";
|
|
@@ -30,6 +33,10 @@ import type { ObjectStorage } from "@opengeni/storage";
|
|
|
30
33
|
import { createHash } from "node:crypto";
|
|
31
34
|
import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
|
|
32
35
|
import type OpenAI from "openai";
|
|
36
|
+
import { KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS } from "@opengeni/contracts";
|
|
37
|
+
import { projectKnowledgeRecord } from "./knowledge-projection";
|
|
38
|
+
|
|
39
|
+
export { projectKnowledgeRecord } from "./knowledge-projection";
|
|
33
40
|
|
|
34
41
|
export const DEFAULT_DOCUMENT_PARSER = "liteparse";
|
|
35
42
|
export const DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
|
|
@@ -198,6 +205,19 @@ export type ListEffectiveIndexedDocumentsInput = {
|
|
|
198
205
|
limit?: number | undefined;
|
|
199
206
|
};
|
|
200
207
|
|
|
208
|
+
export type EffectiveKnowledgeBrowseInput = {
|
|
209
|
+
accountId: string;
|
|
210
|
+
workspaceId: string;
|
|
211
|
+
/** Immutable human subject accepted for the logical request/turn. */
|
|
212
|
+
initiatingSubjectId: string;
|
|
213
|
+
/** Omit to browse top-level documents; pass a document record id for chunks. */
|
|
214
|
+
parentId?: string | undefined;
|
|
215
|
+
topic?: string | undefined;
|
|
216
|
+
sourceKinds?: KnowledgeSourceKind[] | undefined;
|
|
217
|
+
cursor?: string | undefined;
|
|
218
|
+
limit?: number | undefined;
|
|
219
|
+
};
|
|
220
|
+
|
|
201
221
|
export type DocumentIndexHooks = {
|
|
202
222
|
beforeEmbed?: (input: {
|
|
203
223
|
accountId: string;
|
|
@@ -1793,6 +1813,414 @@ export async function searchEffectiveDocuments(
|
|
|
1793
1813
|
);
|
|
1794
1814
|
}
|
|
1795
1815
|
|
|
1816
|
+
/**
|
|
1817
|
+
* Agent-facing Knowledge search over the existing Documents evidence store.
|
|
1818
|
+
* Search selects candidates only after authority filtering; this second read
|
|
1819
|
+
* rechecks every exact chunk before projecting it, so a record revoked between
|
|
1820
|
+
* ranking and response construction disappears rather than leaking stale data.
|
|
1821
|
+
*/
|
|
1822
|
+
export async function searchEffectiveKnowledge(
|
|
1823
|
+
db: Database,
|
|
1824
|
+
input: EffectiveDocumentSearchInput,
|
|
1825
|
+
services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
|
|
1826
|
+
): Promise<KnowledgeSearchResponse> {
|
|
1827
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1828
|
+
const ranked = await searchEffectiveDocuments(
|
|
1829
|
+
db,
|
|
1830
|
+
{ ...input, initiatingSubjectId, surface: "agent" },
|
|
1831
|
+
services,
|
|
1832
|
+
);
|
|
1833
|
+
if (ranked.length === 0) return { results: [] };
|
|
1834
|
+
const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1835
|
+
const current = await withDocumentAccountRls(
|
|
1836
|
+
db,
|
|
1837
|
+
input.accountId,
|
|
1838
|
+
input.workspaceId,
|
|
1839
|
+
access,
|
|
1840
|
+
async (scopedDb) =>
|
|
1841
|
+
await scopedDb
|
|
1842
|
+
.select({ chunk: schema.documentChunks, document: schema.documents })
|
|
1843
|
+
.from(schema.documentChunks)
|
|
1844
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1845
|
+
.where(
|
|
1846
|
+
and(
|
|
1847
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1848
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1849
|
+
inArray(
|
|
1850
|
+
schema.documentChunks.id,
|
|
1851
|
+
ranked.map((result) => result.chunkId),
|
|
1852
|
+
),
|
|
1853
|
+
eq(schema.documents.status, "ready"),
|
|
1854
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
1855
|
+
),
|
|
1856
|
+
),
|
|
1857
|
+
);
|
|
1858
|
+
const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
|
|
1859
|
+
return {
|
|
1860
|
+
results: ranked.flatMap((rankedResult) => {
|
|
1861
|
+
const row = currentByChunkId.get(rankedResult.chunkId);
|
|
1862
|
+
if (!row) return [];
|
|
1863
|
+
return [
|
|
1864
|
+
{
|
|
1865
|
+
record: knowledgeChunkRecord(row.document, row.chunk),
|
|
1866
|
+
retrieval: {
|
|
1867
|
+
score: rankedResult.score,
|
|
1868
|
+
matchType: rankedResult.matchType,
|
|
1869
|
+
vectorScore: rankedResult.vectorScore,
|
|
1870
|
+
keywordScore: rankedResult.keywordScore,
|
|
1871
|
+
},
|
|
1872
|
+
},
|
|
1873
|
+
];
|
|
1874
|
+
}),
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
/** Fetch one stable Knowledge record with a fresh authorization check. */
|
|
1879
|
+
export async function getEffectiveKnowledgeRecord(
|
|
1880
|
+
db: Database,
|
|
1881
|
+
input: {
|
|
1882
|
+
accountId: string;
|
|
1883
|
+
workspaceId: string;
|
|
1884
|
+
initiatingSubjectId: string;
|
|
1885
|
+
id: string;
|
|
1886
|
+
},
|
|
1887
|
+
): Promise<KnowledgeRecord | null> {
|
|
1888
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1889
|
+
const target = parseKnowledgeRecordId(input.id);
|
|
1890
|
+
const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1891
|
+
return await withDocumentAccountRls(
|
|
1892
|
+
db,
|
|
1893
|
+
input.accountId,
|
|
1894
|
+
input.workspaceId,
|
|
1895
|
+
access,
|
|
1896
|
+
async (scopedDb) => {
|
|
1897
|
+
if (target.kind === "document") {
|
|
1898
|
+
const [document] = await scopedDb
|
|
1899
|
+
.select()
|
|
1900
|
+
.from(schema.documents)
|
|
1901
|
+
.where(
|
|
1902
|
+
and(
|
|
1903
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1904
|
+
eq(schema.documents.id, target.id),
|
|
1905
|
+
eq(schema.documents.status, "ready"),
|
|
1906
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
1907
|
+
),
|
|
1908
|
+
)
|
|
1909
|
+
.limit(1);
|
|
1910
|
+
return document ? knowledgeDocumentRecord(document) : null;
|
|
1911
|
+
}
|
|
1912
|
+
const [row] = await scopedDb
|
|
1913
|
+
.select({ chunk: schema.documentChunks, document: schema.documents })
|
|
1914
|
+
.from(schema.documentChunks)
|
|
1915
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1916
|
+
.where(
|
|
1917
|
+
and(
|
|
1918
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1919
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1920
|
+
eq(schema.documentChunks.id, target.id),
|
|
1921
|
+
eq(schema.documents.status, "ready"),
|
|
1922
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
1923
|
+
),
|
|
1924
|
+
)
|
|
1925
|
+
.limit(1);
|
|
1926
|
+
return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
|
|
1927
|
+
},
|
|
1928
|
+
);
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
/**
|
|
1932
|
+
* Browse top-level authorized documents or the chunks of one authorized
|
|
1933
|
+
* document. The cursor is opaque and bound to the caller, workspace, parent,
|
|
1934
|
+
* and filters. It can change pagination position but can never widen scope.
|
|
1935
|
+
*/
|
|
1936
|
+
export async function browseEffectiveKnowledge(
|
|
1937
|
+
db: Database,
|
|
1938
|
+
input: EffectiveKnowledgeBrowseInput,
|
|
1939
|
+
): Promise<KnowledgeBrowseResponse> {
|
|
1940
|
+
const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
|
|
1941
|
+
const limit = input.limit ?? 20;
|
|
1942
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 50) {
|
|
1943
|
+
throw new Error("knowledge browse limit must be between 1 and 50");
|
|
1944
|
+
}
|
|
1945
|
+
const topic = input.topic === undefined ? null : (cleanString(input.topic) ?? null);
|
|
1946
|
+
if (input.topic !== undefined && (!topic || topic !== input.topic || topic.length > 256)) {
|
|
1947
|
+
throw new Error("knowledge browse topic is invalid");
|
|
1948
|
+
}
|
|
1949
|
+
const sourceKinds = [...new Set(input.sourceKinds ?? [])].sort();
|
|
1950
|
+
const parent = input.parentId ? parseKnowledgeRecordId(input.parentId) : null;
|
|
1951
|
+
if (parent?.kind === "document_chunk") {
|
|
1952
|
+
throw new Error("knowledge browse parent must be a document record");
|
|
1953
|
+
}
|
|
1954
|
+
if (parent && (topic || sourceKinds.length > 0)) {
|
|
1955
|
+
throw new Error("knowledge browse document contents do not accept topic/source filters");
|
|
1956
|
+
}
|
|
1957
|
+
const cursorScope = {
|
|
1958
|
+
accountId: input.accountId,
|
|
1959
|
+
workspaceId: input.workspaceId,
|
|
1960
|
+
initiatingSubjectId,
|
|
1961
|
+
parentId: parent ? `${parent.kind}:${parent.id}` : null,
|
|
1962
|
+
topic,
|
|
1963
|
+
sourceKinds,
|
|
1964
|
+
};
|
|
1965
|
+
const after = input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
|
|
1966
|
+
if (parent && after > 2_147_483_648n) {
|
|
1967
|
+
throw new Error("invalid knowledge browse cursor");
|
|
1968
|
+
}
|
|
1969
|
+
const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
|
|
1970
|
+
return await withDocumentAccountRls(
|
|
1971
|
+
db,
|
|
1972
|
+
input.accountId,
|
|
1973
|
+
input.workspaceId,
|
|
1974
|
+
access,
|
|
1975
|
+
async (scopedDb) => {
|
|
1976
|
+
if (parent) {
|
|
1977
|
+
const [authorizedParent] = await scopedDb
|
|
1978
|
+
.select({ id: schema.documents.id })
|
|
1979
|
+
.from(schema.documents)
|
|
1980
|
+
.where(
|
|
1981
|
+
and(
|
|
1982
|
+
eq(schema.documents.accountId, input.accountId),
|
|
1983
|
+
eq(schema.documents.id, parent.id),
|
|
1984
|
+
eq(schema.documents.status, "ready"),
|
|
1985
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
1986
|
+
),
|
|
1987
|
+
)
|
|
1988
|
+
.limit(1);
|
|
1989
|
+
if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
|
|
1990
|
+
const rows = await scopedDb
|
|
1991
|
+
.select({ chunk: schema.documentChunks, document: schema.documents })
|
|
1992
|
+
.from(schema.documentChunks)
|
|
1993
|
+
.innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
|
|
1994
|
+
.where(
|
|
1995
|
+
and(
|
|
1996
|
+
eq(schema.documentChunks.accountId, input.accountId),
|
|
1997
|
+
eq(schema.documentChunks.documentId, parent.id),
|
|
1998
|
+
gt(schema.documentChunks.chunkIndex, Number(after) - 1),
|
|
1999
|
+
eq(schema.documents.status, "ready"),
|
|
2000
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
2001
|
+
),
|
|
2002
|
+
)
|
|
2003
|
+
.orderBy(asc(schema.documentChunks.chunkIndex))
|
|
2004
|
+
.limit(limit + 1);
|
|
2005
|
+
const hasMore = rows.length > limit;
|
|
2006
|
+
const page = rows.slice(0, limit);
|
|
2007
|
+
const last = page.at(-1)?.chunk.chunkIndex;
|
|
2008
|
+
return {
|
|
2009
|
+
records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
|
|
2010
|
+
nextCursor:
|
|
2011
|
+
hasMore && last !== undefined
|
|
2012
|
+
? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last + 1))
|
|
2013
|
+
: null,
|
|
2014
|
+
hasMore,
|
|
2015
|
+
};
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
const conditions: SQL[] = [
|
|
2019
|
+
eq(schema.documents.accountId, input.accountId),
|
|
2020
|
+
eq(schema.documents.status, "ready"),
|
|
2021
|
+
isNotNull(schema.documents.indexSequence),
|
|
2022
|
+
gt(schema.documents.indexSequence, after),
|
|
2023
|
+
...documentAccessConditions(input.workspaceId, access),
|
|
2024
|
+
];
|
|
2025
|
+
if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
|
|
2026
|
+
if (sourceKinds.length > 0)
|
|
2027
|
+
conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
|
|
2028
|
+
const rows = await scopedDb
|
|
2029
|
+
.select()
|
|
2030
|
+
.from(schema.documents)
|
|
2031
|
+
.where(and(...conditions))
|
|
2032
|
+
.orderBy(asc(schema.documents.indexSequence))
|
|
2033
|
+
.limit(limit + 1);
|
|
2034
|
+
const hasMore = rows.length > limit;
|
|
2035
|
+
const page = rows.slice(0, limit);
|
|
2036
|
+
const last = page.at(-1)?.indexSequence;
|
|
2037
|
+
return {
|
|
2038
|
+
records: page.map(knowledgeDocumentRecord),
|
|
2039
|
+
nextCursor:
|
|
2040
|
+
hasMore && last !== undefined && last !== null
|
|
2041
|
+
? encodeKnowledgeBrowseCursor(cursorScope, last)
|
|
2042
|
+
: null,
|
|
2043
|
+
hasMore,
|
|
2044
|
+
};
|
|
2045
|
+
},
|
|
2046
|
+
);
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
type KnowledgeBrowseCursorScope = {
|
|
2050
|
+
accountId: string;
|
|
2051
|
+
workspaceId: string;
|
|
2052
|
+
initiatingSubjectId: string;
|
|
2053
|
+
parentId: string | null;
|
|
2054
|
+
topic: string | null;
|
|
2055
|
+
sourceKinds: readonly string[];
|
|
2056
|
+
};
|
|
2057
|
+
|
|
2058
|
+
export function encodeKnowledgeBrowseCursor(
|
|
2059
|
+
scope: KnowledgeBrowseCursorScope,
|
|
2060
|
+
position: bigint,
|
|
2061
|
+
): string {
|
|
2062
|
+
if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
|
|
2063
|
+
return Buffer.from(
|
|
2064
|
+
JSON.stringify({ v: 1, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
|
|
2065
|
+
"utf8",
|
|
2066
|
+
).toString("base64url");
|
|
2067
|
+
}
|
|
2068
|
+
|
|
2069
|
+
export function decodeKnowledgeBrowseCursor(
|
|
2070
|
+
value: string,
|
|
2071
|
+
scope: KnowledgeBrowseCursorScope,
|
|
2072
|
+
): bigint {
|
|
2073
|
+
try {
|
|
2074
|
+
if (!value || value.length > KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS) {
|
|
2075
|
+
throw new Error("cursor length");
|
|
2076
|
+
}
|
|
2077
|
+
const bytes = Buffer.from(value, "base64url");
|
|
2078
|
+
if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
|
|
2079
|
+
const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
|
2080
|
+
if (
|
|
2081
|
+
Object.keys(parsed).sort().join(",") !== "q,s,v" ||
|
|
2082
|
+
parsed.v !== 1 ||
|
|
2083
|
+
typeof parsed.s !== "string" ||
|
|
2084
|
+
typeof parsed.q !== "string" ||
|
|
2085
|
+
!/^(0|[1-9][0-9]*)$/.test(parsed.q)
|
|
2086
|
+
) {
|
|
2087
|
+
throw new Error("cursor payload");
|
|
2088
|
+
}
|
|
2089
|
+
if (parsed.s !== knowledgeBrowseCursorScope(scope)) {
|
|
2090
|
+
throw new Error("knowledge browse cursor belongs to a different scope");
|
|
2091
|
+
}
|
|
2092
|
+
const position = BigInt(parsed.q);
|
|
2093
|
+
if (position > 9_223_372_036_854_775_807n) throw new Error("cursor position");
|
|
2094
|
+
return position;
|
|
2095
|
+
} catch (error) {
|
|
2096
|
+
if (
|
|
2097
|
+
error instanceof Error &&
|
|
2098
|
+
error.message === "knowledge browse cursor belongs to a different scope"
|
|
2099
|
+
) {
|
|
2100
|
+
throw error;
|
|
2101
|
+
}
|
|
2102
|
+
throw new Error("invalid knowledge browse cursor", { cause: error });
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
|
|
2107
|
+
return createHash("sha256")
|
|
2108
|
+
.update("opengeni:knowledge-browse-cursor:v1\0")
|
|
2109
|
+
.update(scope.accountId)
|
|
2110
|
+
.update("\0")
|
|
2111
|
+
.update(scope.workspaceId)
|
|
2112
|
+
.update("\0")
|
|
2113
|
+
.update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
|
|
2114
|
+
.update("\0")
|
|
2115
|
+
.update(scope.parentId ?? "")
|
|
2116
|
+
.update("\0")
|
|
2117
|
+
.update(scope.topic ?? "")
|
|
2118
|
+
.update("\0")
|
|
2119
|
+
.update([...scope.sourceKinds].sort().join("\0"))
|
|
2120
|
+
.digest("hex");
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
function parseKnowledgeRecordId(value: string): {
|
|
2124
|
+
kind: "document" | "document_chunk";
|
|
2125
|
+
id: string;
|
|
2126
|
+
} {
|
|
2127
|
+
const match =
|
|
2128
|
+
/^(document|document_chunk):([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/iu.exec(
|
|
2129
|
+
value,
|
|
2130
|
+
);
|
|
2131
|
+
if (!match) throw new Error("invalid knowledge record id");
|
|
2132
|
+
return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect): KnowledgeRecord {
|
|
2136
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2137
|
+
const projected = projectKnowledgeRecord({
|
|
2138
|
+
title: document.title,
|
|
2139
|
+
body: document.summary,
|
|
2140
|
+
summary: document.summary,
|
|
2141
|
+
topics: document.topics,
|
|
2142
|
+
metadata: { parser: document.parser, chunkCount: document.chunkCount },
|
|
2143
|
+
source: knowledgeSource(document),
|
|
2144
|
+
});
|
|
2145
|
+
return {
|
|
2146
|
+
id: `document:${document.id}`,
|
|
2147
|
+
kind: "document",
|
|
2148
|
+
title: projected.title,
|
|
2149
|
+
content: projected.content,
|
|
2150
|
+
authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
|
|
2151
|
+
provenance: {
|
|
2152
|
+
source: projected.source,
|
|
2153
|
+
indexedAt: document.indexedAt.toISOString(),
|
|
2154
|
+
},
|
|
2155
|
+
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2156
|
+
quality: knowledgeQuality(document),
|
|
2157
|
+
links: knowledgeSourceLinks(projected.source.uri),
|
|
2158
|
+
projection: projected.projection,
|
|
2159
|
+
};
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
function knowledgeChunkRecord(
|
|
2163
|
+
document: typeof schema.documents.$inferSelect,
|
|
2164
|
+
chunk: typeof schema.documentChunks.$inferSelect,
|
|
2165
|
+
): KnowledgeRecord {
|
|
2166
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2167
|
+
const projected = projectKnowledgeRecord({
|
|
2168
|
+
title: document.title,
|
|
2169
|
+
body: chunk.text,
|
|
2170
|
+
summary: document.summary,
|
|
2171
|
+
topics: document.topics,
|
|
2172
|
+
metadata: { ...chunk.metadata, chunkIndex: chunk.chunkIndex },
|
|
2173
|
+
source: knowledgeSource(document),
|
|
2174
|
+
});
|
|
2175
|
+
return {
|
|
2176
|
+
id: `document_chunk:${chunk.id}`,
|
|
2177
|
+
kind: "document_chunk",
|
|
2178
|
+
title: projected.title,
|
|
2179
|
+
content: projected.content,
|
|
2180
|
+
authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
|
|
2181
|
+
provenance: {
|
|
2182
|
+
source: projected.source,
|
|
2183
|
+
indexedAt: document.indexedAt.toISOString(),
|
|
2184
|
+
},
|
|
2185
|
+
lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
|
|
2186
|
+
quality: knowledgeQuality(document),
|
|
2187
|
+
links: [
|
|
2188
|
+
{ relation: "parent", target: { kind: "knowledge", id: `document:${document.id}` } },
|
|
2189
|
+
...knowledgeSourceLinks(projected.source.uri),
|
|
2190
|
+
],
|
|
2191
|
+
projection: projected.projection,
|
|
2192
|
+
};
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
function knowledgeSource(document: typeof schema.documents.$inferSelect) {
|
|
2196
|
+
return {
|
|
2197
|
+
kind: normalizeKnowledgeSourceKind(document.sourceKind),
|
|
2198
|
+
uri: document.sourceUri,
|
|
2199
|
+
externalId: document.sourceExternalId,
|
|
2200
|
+
title: document.sourceTitle,
|
|
2201
|
+
author: document.sourceAuthor,
|
|
2202
|
+
createdAt: document.sourceCreatedAt?.toISOString() ?? null,
|
|
2203
|
+
updatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
|
|
2204
|
+
version: document.sourceVersion,
|
|
2205
|
+
};
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
function knowledgeQuality(
|
|
2209
|
+
document: typeof schema.documents.$inferSelect,
|
|
2210
|
+
): KnowledgeRecord["quality"] {
|
|
2211
|
+
if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
|
|
2212
|
+
return {
|
|
2213
|
+
trust: "sourced",
|
|
2214
|
+
freshnessAt: (document.sourceUpdatedAt ?? document.indexedAt).toISOString(),
|
|
2215
|
+
conflict: "not_evaluated",
|
|
2216
|
+
correction: "current_source_version",
|
|
2217
|
+
};
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"] {
|
|
2221
|
+
return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
|
|
2222
|
+
}
|
|
2223
|
+
|
|
1796
2224
|
async function vectorSearchDocuments(
|
|
1797
2225
|
db: Database,
|
|
1798
2226
|
input: DocumentSearchInput,
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import {
|
|
2
|
+
KNOWLEDGE_BODY_MAX_BYTES,
|
|
3
|
+
KNOWLEDGE_METADATA_MAX_BYTES,
|
|
4
|
+
KNOWLEDGE_METADATA_MAX_DEPTH,
|
|
5
|
+
KNOWLEDGE_METADATA_MAX_ITEMS,
|
|
6
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
7
|
+
KNOWLEDGE_SOURCE_URI_MAX_BYTES,
|
|
8
|
+
KNOWLEDGE_SUMMARY_MAX_BYTES,
|
|
9
|
+
KNOWLEDGE_TITLE_MAX_BYTES,
|
|
10
|
+
KNOWLEDGE_TOPIC_MAX_BYTES,
|
|
11
|
+
KNOWLEDGE_TOPICS_MAX_ITEMS,
|
|
12
|
+
type KnowledgeRecord,
|
|
13
|
+
type KnowledgeSource,
|
|
14
|
+
} from "@opengeni/contracts";
|
|
15
|
+
|
|
16
|
+
type ProjectionField = KnowledgeRecord["projection"]["fields"][number];
|
|
17
|
+
|
|
18
|
+
export type KnowledgeProjectionInput = {
|
|
19
|
+
title: string;
|
|
20
|
+
body: string | null;
|
|
21
|
+
summary: string | null;
|
|
22
|
+
topics: unknown;
|
|
23
|
+
metadata: unknown;
|
|
24
|
+
source: KnowledgeSource;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type KnowledgeProjectionResult = Pick<
|
|
28
|
+
KnowledgeRecord,
|
|
29
|
+
"title" | "content" | "projection"
|
|
30
|
+
> & {
|
|
31
|
+
source: KnowledgeSource;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const utf8Bytes = (value: string): number => Buffer.byteLength(value, "utf8");
|
|
35
|
+
|
|
36
|
+
function truncateUtf8(value: string, maxBytes: number): { value: string; truncated: boolean } {
|
|
37
|
+
if (utf8Bytes(value) <= maxBytes) return { value, truncated: false };
|
|
38
|
+
let low = 0;
|
|
39
|
+
let high = value.length;
|
|
40
|
+
while (low < high) {
|
|
41
|
+
const middle = Math.ceil((low + high) / 2);
|
|
42
|
+
const end =
|
|
43
|
+
middle > 0 && middle < value.length && /[\uD800-\uDBFF]/u.test(value[middle - 1]!)
|
|
44
|
+
? middle - 1
|
|
45
|
+
: middle;
|
|
46
|
+
if (utf8Bytes(value.slice(0, end)) <= maxBytes) low = middle;
|
|
47
|
+
else high = middle - 1;
|
|
48
|
+
}
|
|
49
|
+
let end = low;
|
|
50
|
+
if (end > 0 && end < value.length && /[\uD800-\uDBFF]/u.test(value[end - 1]!)) end -= 1;
|
|
51
|
+
while (end > 0 && utf8Bytes(value.slice(0, end)) > maxBytes) end -= 1;
|
|
52
|
+
return { value: value.slice(0, end), truncated: true };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function projectNullableString(
|
|
56
|
+
value: string | null,
|
|
57
|
+
maxBytes: number,
|
|
58
|
+
field: ProjectionField,
|
|
59
|
+
fields: Set<ProjectionField>,
|
|
60
|
+
): string | null {
|
|
61
|
+
if (value === null) return null;
|
|
62
|
+
const projected = truncateUtf8(value, maxBytes);
|
|
63
|
+
if (projected.truncated) fields.add(field);
|
|
64
|
+
return projected.value;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function projectSourceUri(value: string | null, fields: Set<ProjectionField>): string | null {
|
|
68
|
+
if (value === null || value === "") return null;
|
|
69
|
+
if (utf8Bytes(value) > KNOWLEDGE_SOURCE_URI_MAX_BYTES) {
|
|
70
|
+
fields.add("provenance.source.uri");
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function projectTopics(value: unknown, fields: Set<ProjectionField>): string[] {
|
|
77
|
+
if (!Array.isArray(value)) {
|
|
78
|
+
if (value !== null && value !== undefined) fields.add("content.topics");
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
81
|
+
const result: string[] = [];
|
|
82
|
+
for (const candidate of value) {
|
|
83
|
+
if (result.length >= KNOWLEDGE_TOPICS_MAX_ITEMS) {
|
|
84
|
+
fields.add("content.topics");
|
|
85
|
+
break;
|
|
86
|
+
}
|
|
87
|
+
if (typeof candidate !== "string") {
|
|
88
|
+
fields.add("content.topics");
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const projected = truncateUtf8(candidate, KNOWLEDGE_TOPIC_MAX_BYTES);
|
|
92
|
+
if (projected.truncated) fields.add("content.topics");
|
|
93
|
+
result.push(projected.value);
|
|
94
|
+
}
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function projectMetadata(value: unknown): { value: Record<string, unknown>; truncated: boolean } {
|
|
99
|
+
let remainingItems = KNOWLEDGE_METADATA_MAX_ITEMS;
|
|
100
|
+
let truncated = false;
|
|
101
|
+
|
|
102
|
+
const visit = (candidate: unknown, depth: number): unknown => {
|
|
103
|
+
if (remainingItems <= 0 || depth > KNOWLEDGE_METADATA_MAX_DEPTH) {
|
|
104
|
+
truncated = true;
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
remainingItems -= 1;
|
|
108
|
+
if (
|
|
109
|
+
candidate === null ||
|
|
110
|
+
typeof candidate === "boolean" ||
|
|
111
|
+
(typeof candidate === "number" && Number.isFinite(candidate))
|
|
112
|
+
) {
|
|
113
|
+
return candidate;
|
|
114
|
+
}
|
|
115
|
+
if (typeof candidate === "string") {
|
|
116
|
+
const projected = truncateUtf8(candidate, KNOWLEDGE_SOURCE_STRING_MAX_BYTES);
|
|
117
|
+
if (projected.truncated) truncated = true;
|
|
118
|
+
return projected.value;
|
|
119
|
+
}
|
|
120
|
+
if (Array.isArray(candidate)) {
|
|
121
|
+
const result: unknown[] = [];
|
|
122
|
+
for (const item of candidate) {
|
|
123
|
+
const projected = visit(item, depth + 1);
|
|
124
|
+
if (projected === undefined) break;
|
|
125
|
+
result.push(projected);
|
|
126
|
+
}
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
if (candidate && typeof candidate === "object") {
|
|
130
|
+
const result: Record<string, unknown> = {};
|
|
131
|
+
for (const key of Object.keys(candidate as Record<string, unknown>).sort()) {
|
|
132
|
+
if (utf8Bytes(key) > KNOWLEDGE_TOPIC_MAX_BYTES) {
|
|
133
|
+
truncated = true;
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
const projected = visit((candidate as Record<string, unknown>)[key], depth + 1);
|
|
137
|
+
if (projected === undefined) break;
|
|
138
|
+
result[key] = projected;
|
|
139
|
+
}
|
|
140
|
+
return result;
|
|
141
|
+
}
|
|
142
|
+
truncated = true;
|
|
143
|
+
return undefined;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const root = visit(value, 0);
|
|
147
|
+
let result = root && typeof root === "object" && !Array.isArray(root) ? root : {};
|
|
148
|
+
if (result !== root) truncated = true;
|
|
149
|
+
const bounded: Record<string, unknown> = {};
|
|
150
|
+
for (const key of Object.keys(result as Record<string, unknown>).sort()) {
|
|
151
|
+
bounded[key] = (result as Record<string, unknown>)[key];
|
|
152
|
+
if (utf8Bytes(JSON.stringify(bounded)) > KNOWLEDGE_METADATA_MAX_BYTES) {
|
|
153
|
+
delete bounded[key];
|
|
154
|
+
truncated = true;
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
result = bounded;
|
|
159
|
+
return { value: result as Record<string, unknown>, truncated };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Deterministic, byte-bounded projection at the agent-facing Knowledge envelope. */
|
|
163
|
+
export function projectKnowledgeRecord(input: KnowledgeProjectionInput): KnowledgeProjectionResult {
|
|
164
|
+
const fields = new Set<ProjectionField>();
|
|
165
|
+
const title = truncateUtf8(input.title, KNOWLEDGE_TITLE_MAX_BYTES);
|
|
166
|
+
if (title.truncated) fields.add("title");
|
|
167
|
+
const metadata = projectMetadata(input.metadata);
|
|
168
|
+
if (metadata.truncated) fields.add("content.metadata");
|
|
169
|
+
const source: KnowledgeSource = {
|
|
170
|
+
...input.source,
|
|
171
|
+
uri: projectSourceUri(input.source.uri, fields),
|
|
172
|
+
externalId: projectNullableString(
|
|
173
|
+
input.source.externalId,
|
|
174
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
175
|
+
"provenance.source.externalId",
|
|
176
|
+
fields,
|
|
177
|
+
),
|
|
178
|
+
title: projectNullableString(
|
|
179
|
+
input.source.title,
|
|
180
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
181
|
+
"provenance.source.title",
|
|
182
|
+
fields,
|
|
183
|
+
),
|
|
184
|
+
author: projectNullableString(
|
|
185
|
+
input.source.author,
|
|
186
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
187
|
+
"provenance.source.author",
|
|
188
|
+
fields,
|
|
189
|
+
),
|
|
190
|
+
version: projectNullableString(
|
|
191
|
+
input.source.version,
|
|
192
|
+
KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
|
|
193
|
+
"provenance.source.version",
|
|
194
|
+
fields,
|
|
195
|
+
),
|
|
196
|
+
};
|
|
197
|
+
return {
|
|
198
|
+
title: title.value,
|
|
199
|
+
content: {
|
|
200
|
+
format: "markdown",
|
|
201
|
+
body: projectNullableString(input.body, KNOWLEDGE_BODY_MAX_BYTES, "content.body", fields),
|
|
202
|
+
summary: projectNullableString(
|
|
203
|
+
input.summary,
|
|
204
|
+
KNOWLEDGE_SUMMARY_MAX_BYTES,
|
|
205
|
+
"content.summary",
|
|
206
|
+
fields,
|
|
207
|
+
),
|
|
208
|
+
topics: projectTopics(input.topics, fields),
|
|
209
|
+
metadata: metadata.value,
|
|
210
|
+
},
|
|
211
|
+
source,
|
|
212
|
+
projection: { truncated: fields.size > 0, fields: [...fields].sort() },
|
|
213
|
+
};
|
|
214
|
+
}
|