@opengeni/documents 0.5.32 → 0.5.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ import { type KnowledgeRecord, type KnowledgeSource } from "@opengeni/contracts";
2
+ export type KnowledgeProjectionInput = {
3
+ title: string;
4
+ body: string | null;
5
+ summary: string | null;
6
+ topics: unknown;
7
+ metadata: unknown;
8
+ source: KnowledgeSource;
9
+ };
10
+ export type KnowledgeProjectionResult = Pick<KnowledgeRecord, "title" | "content" | "projection"> & {
11
+ source: KnowledgeSource;
12
+ };
13
+ /** Deterministic, byte-bounded projection at the agent-facing Knowledge envelope. */
14
+ export declare function projectKnowledgeRecord(input: KnowledgeProjectionInput): KnowledgeProjectionResult;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/documents",
3
- "version": "0.5.32",
3
+ "version": "0.5.38",
4
4
  "license": "Apache-2.0",
5
5
  "repository": {
6
6
  "type": "git",
@@ -40,10 +40,10 @@
40
40
  },
41
41
  "dependencies": {
42
42
  "@llamaindex/liteparse": "^1.5.3",
43
- "@opengeni/config": "^0.13.2",
44
- "@opengeni/contracts": "^0.44.1",
45
- "@opengeni/db": "^0.31.1",
46
- "@opengeni/storage": "^0.2.87",
43
+ "@opengeni/config": "^0.16.1",
44
+ "@opengeni/contracts": "^0.50.0",
45
+ "@opengeni/db": "^0.36.0",
46
+ "@opengeni/storage": "^0.2.93",
47
47
  "drizzle-orm": "^0.45.2",
48
48
  "openai": "6.47.0"
49
49
  }
package/src/index.ts CHANGED
@@ -13,6 +13,9 @@ import type {
13
13
  DocumentVisibility,
14
14
  FileAsset,
15
15
  IndexedDocumentSummary,
16
+ KnowledgeBrowseResponse,
17
+ KnowledgeRecord,
18
+ KnowledgeSearchResponse,
16
19
  KnowledgeSourceKind,
17
20
  ListIndexedDocumentsResponse,
18
21
  } from "@opengeni/contracts";
@@ -30,6 +33,10 @@ import type { ObjectStorage } from "@opengeni/storage";
30
33
  import { createHash } from "node:crypto";
31
34
  import { and, asc, desc, eq, gt, inArray, isNotNull, or, sql, type SQL } from "drizzle-orm";
32
35
  import type OpenAI from "openai";
36
+ import { KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS } from "@opengeni/contracts";
37
+ import { projectKnowledgeRecord } from "./knowledge-projection";
38
+
39
+ export { projectKnowledgeRecord } from "./knowledge-projection";
33
40
 
34
41
  export const DEFAULT_DOCUMENT_PARSER = "liteparse";
35
42
  export const DEFAULT_DOCUMENT_EMBEDDING_MODEL = "text-embedding-3-large";
@@ -198,6 +205,19 @@ export type ListEffectiveIndexedDocumentsInput = {
198
205
  limit?: number | undefined;
199
206
  };
200
207
 
208
+ export type EffectiveKnowledgeBrowseInput = {
209
+ accountId: string;
210
+ workspaceId: string;
211
+ /** Immutable human subject accepted for the logical request/turn. */
212
+ initiatingSubjectId: string;
213
+ /** Omit to browse top-level documents; pass a document record id for chunks. */
214
+ parentId?: string | undefined;
215
+ topic?: string | undefined;
216
+ sourceKinds?: KnowledgeSourceKind[] | undefined;
217
+ cursor?: string | undefined;
218
+ limit?: number | undefined;
219
+ };
220
+
201
221
  export type DocumentIndexHooks = {
202
222
  beforeEmbed?: (input: {
203
223
  accountId: string;
@@ -1793,6 +1813,414 @@ export async function searchEffectiveDocuments(
1793
1813
  );
1794
1814
  }
1795
1815
 
1816
+ /**
1817
+ * Agent-facing Knowledge search over the existing Documents evidence store.
1818
+ * Search selects candidates only after authority filtering; this second read
1819
+ * rechecks every exact chunk before projecting it, so a record revoked between
1820
+ * ranking and response construction disappears rather than leaking stale data.
1821
+ */
1822
+ export async function searchEffectiveKnowledge(
1823
+ db: Database,
1824
+ input: EffectiveDocumentSearchInput,
1825
+ services: Pick<DocumentServices, "embedder"> = createDocumentServices(),
1826
+ ): Promise<KnowledgeSearchResponse> {
1827
+ const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1828
+ const ranked = await searchEffectiveDocuments(
1829
+ db,
1830
+ { ...input, initiatingSubjectId, surface: "agent" },
1831
+ services,
1832
+ );
1833
+ if (ranked.length === 0) return { results: [] };
1834
+ const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
1835
+ const current = await withDocumentAccountRls(
1836
+ db,
1837
+ input.accountId,
1838
+ input.workspaceId,
1839
+ access,
1840
+ async (scopedDb) =>
1841
+ await scopedDb
1842
+ .select({ chunk: schema.documentChunks, document: schema.documents })
1843
+ .from(schema.documentChunks)
1844
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1845
+ .where(
1846
+ and(
1847
+ eq(schema.documents.accountId, input.accountId),
1848
+ eq(schema.documentChunks.accountId, input.accountId),
1849
+ inArray(
1850
+ schema.documentChunks.id,
1851
+ ranked.map((result) => result.chunkId),
1852
+ ),
1853
+ eq(schema.documents.status, "ready"),
1854
+ ...documentAccessConditions(input.workspaceId, access),
1855
+ ),
1856
+ ),
1857
+ );
1858
+ const currentByChunkId = new Map(current.map((row) => [row.chunk.id, row]));
1859
+ return {
1860
+ results: ranked.flatMap((rankedResult) => {
1861
+ const row = currentByChunkId.get(rankedResult.chunkId);
1862
+ if (!row) return [];
1863
+ return [
1864
+ {
1865
+ record: knowledgeChunkRecord(row.document, row.chunk),
1866
+ retrieval: {
1867
+ score: rankedResult.score,
1868
+ matchType: rankedResult.matchType,
1869
+ vectorScore: rankedResult.vectorScore,
1870
+ keywordScore: rankedResult.keywordScore,
1871
+ },
1872
+ },
1873
+ ];
1874
+ }),
1875
+ };
1876
+ }
1877
+
1878
+ /** Fetch one stable Knowledge record with a fresh authorization check. */
1879
+ export async function getEffectiveKnowledgeRecord(
1880
+ db: Database,
1881
+ input: {
1882
+ accountId: string;
1883
+ workspaceId: string;
1884
+ initiatingSubjectId: string;
1885
+ id: string;
1886
+ },
1887
+ ): Promise<KnowledgeRecord | null> {
1888
+ const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1889
+ const target = parseKnowledgeRecordId(input.id);
1890
+ const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
1891
+ return await withDocumentAccountRls(
1892
+ db,
1893
+ input.accountId,
1894
+ input.workspaceId,
1895
+ access,
1896
+ async (scopedDb) => {
1897
+ if (target.kind === "document") {
1898
+ const [document] = await scopedDb
1899
+ .select()
1900
+ .from(schema.documents)
1901
+ .where(
1902
+ and(
1903
+ eq(schema.documents.accountId, input.accountId),
1904
+ eq(schema.documents.id, target.id),
1905
+ eq(schema.documents.status, "ready"),
1906
+ ...documentAccessConditions(input.workspaceId, access),
1907
+ ),
1908
+ )
1909
+ .limit(1);
1910
+ return document ? knowledgeDocumentRecord(document) : null;
1911
+ }
1912
+ const [row] = await scopedDb
1913
+ .select({ chunk: schema.documentChunks, document: schema.documents })
1914
+ .from(schema.documentChunks)
1915
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1916
+ .where(
1917
+ and(
1918
+ eq(schema.documents.accountId, input.accountId),
1919
+ eq(schema.documentChunks.accountId, input.accountId),
1920
+ eq(schema.documentChunks.id, target.id),
1921
+ eq(schema.documents.status, "ready"),
1922
+ ...documentAccessConditions(input.workspaceId, access),
1923
+ ),
1924
+ )
1925
+ .limit(1);
1926
+ return row ? knowledgeChunkRecord(row.document, row.chunk) : null;
1927
+ },
1928
+ );
1929
+ }
1930
+
1931
+ /**
1932
+ * Browse top-level authorized documents or the chunks of one authorized
1933
+ * document. The cursor is opaque and bound to the caller, workspace, parent,
1934
+ * and filters. It can change pagination position but can never widen scope.
1935
+ */
1936
+ export async function browseEffectiveKnowledge(
1937
+ db: Database,
1938
+ input: EffectiveKnowledgeBrowseInput,
1939
+ ): Promise<KnowledgeBrowseResponse> {
1940
+ const initiatingSubjectId = canonicalEffectiveDocumentSubject(input.initiatingSubjectId);
1941
+ const limit = input.limit ?? 20;
1942
+ if (!Number.isInteger(limit) || limit < 1 || limit > 50) {
1943
+ throw new Error("knowledge browse limit must be between 1 and 50");
1944
+ }
1945
+ const topic = input.topic === undefined ? null : (cleanString(input.topic) ?? null);
1946
+ if (input.topic !== undefined && (!topic || topic !== input.topic || topic.length > 256)) {
1947
+ throw new Error("knowledge browse topic is invalid");
1948
+ }
1949
+ const sourceKinds = [...new Set(input.sourceKinds ?? [])].sort();
1950
+ const parent = input.parentId ? parseKnowledgeRecordId(input.parentId) : null;
1951
+ if (parent?.kind === "document_chunk") {
1952
+ throw new Error("knowledge browse parent must be a document record");
1953
+ }
1954
+ if (parent && (topic || sourceKinds.length > 0)) {
1955
+ throw new Error("knowledge browse document contents do not accept topic/source filters");
1956
+ }
1957
+ const cursorScope = {
1958
+ accountId: input.accountId,
1959
+ workspaceId: input.workspaceId,
1960
+ initiatingSubjectId,
1961
+ parentId: parent ? `${parent.kind}:${parent.id}` : null,
1962
+ topic,
1963
+ sourceKinds,
1964
+ };
1965
+ const after = input.cursor ? decodeKnowledgeBrowseCursor(input.cursor, cursorScope) : 0n;
1966
+ if (parent && after > 2_147_483_648n) {
1967
+ throw new Error("invalid knowledge browse cursor");
1968
+ }
1969
+ const access: DocumentAccessFilter = { agentOnly: true, viewerSubjectId: initiatingSubjectId };
1970
+ return await withDocumentAccountRls(
1971
+ db,
1972
+ input.accountId,
1973
+ input.workspaceId,
1974
+ access,
1975
+ async (scopedDb) => {
1976
+ if (parent) {
1977
+ const [authorizedParent] = await scopedDb
1978
+ .select({ id: schema.documents.id })
1979
+ .from(schema.documents)
1980
+ .where(
1981
+ and(
1982
+ eq(schema.documents.accountId, input.accountId),
1983
+ eq(schema.documents.id, parent.id),
1984
+ eq(schema.documents.status, "ready"),
1985
+ ...documentAccessConditions(input.workspaceId, access),
1986
+ ),
1987
+ )
1988
+ .limit(1);
1989
+ if (!authorizedParent) return { records: [], nextCursor: null, hasMore: false };
1990
+ const rows = await scopedDb
1991
+ .select({ chunk: schema.documentChunks, document: schema.documents })
1992
+ .from(schema.documentChunks)
1993
+ .innerJoin(schema.documents, eq(schema.documentChunks.documentId, schema.documents.id))
1994
+ .where(
1995
+ and(
1996
+ eq(schema.documentChunks.accountId, input.accountId),
1997
+ eq(schema.documentChunks.documentId, parent.id),
1998
+ gt(schema.documentChunks.chunkIndex, Number(after) - 1),
1999
+ eq(schema.documents.status, "ready"),
2000
+ ...documentAccessConditions(input.workspaceId, access),
2001
+ ),
2002
+ )
2003
+ .orderBy(asc(schema.documentChunks.chunkIndex))
2004
+ .limit(limit + 1);
2005
+ const hasMore = rows.length > limit;
2006
+ const page = rows.slice(0, limit);
2007
+ const last = page.at(-1)?.chunk.chunkIndex;
2008
+ return {
2009
+ records: page.map((row) => knowledgeChunkRecord(row.document, row.chunk)),
2010
+ nextCursor:
2011
+ hasMore && last !== undefined
2012
+ ? encodeKnowledgeBrowseCursor(cursorScope, BigInt(last + 1))
2013
+ : null,
2014
+ hasMore,
2015
+ };
2016
+ }
2017
+
2018
+ const conditions: SQL[] = [
2019
+ eq(schema.documents.accountId, input.accountId),
2020
+ eq(schema.documents.status, "ready"),
2021
+ isNotNull(schema.documents.indexSequence),
2022
+ gt(schema.documents.indexSequence, after),
2023
+ ...documentAccessConditions(input.workspaceId, access),
2024
+ ];
2025
+ if (topic) conditions.push(sql`${schema.documents.topics} ? ${topic}`);
2026
+ if (sourceKinds.length > 0)
2027
+ conditions.push(inArray(schema.documents.sourceKind, sourceKinds));
2028
+ const rows = await scopedDb
2029
+ .select()
2030
+ .from(schema.documents)
2031
+ .where(and(...conditions))
2032
+ .orderBy(asc(schema.documents.indexSequence))
2033
+ .limit(limit + 1);
2034
+ const hasMore = rows.length > limit;
2035
+ const page = rows.slice(0, limit);
2036
+ const last = page.at(-1)?.indexSequence;
2037
+ return {
2038
+ records: page.map(knowledgeDocumentRecord),
2039
+ nextCursor:
2040
+ hasMore && last !== undefined && last !== null
2041
+ ? encodeKnowledgeBrowseCursor(cursorScope, last)
2042
+ : null,
2043
+ hasMore,
2044
+ };
2045
+ },
2046
+ );
2047
+ }
2048
+
2049
+ type KnowledgeBrowseCursorScope = {
2050
+ accountId: string;
2051
+ workspaceId: string;
2052
+ initiatingSubjectId: string;
2053
+ parentId: string | null;
2054
+ topic: string | null;
2055
+ sourceKinds: readonly string[];
2056
+ };
2057
+
2058
+ export function encodeKnowledgeBrowseCursor(
2059
+ scope: KnowledgeBrowseCursorScope,
2060
+ position: bigint,
2061
+ ): string {
2062
+ if (position < 0n) throw new Error("knowledge browse cursor position is invalid");
2063
+ return Buffer.from(
2064
+ JSON.stringify({ v: 1, s: knowledgeBrowseCursorScope(scope), q: position.toString() }),
2065
+ "utf8",
2066
+ ).toString("base64url");
2067
+ }
2068
+
2069
+ export function decodeKnowledgeBrowseCursor(
2070
+ value: string,
2071
+ scope: KnowledgeBrowseCursorScope,
2072
+ ): bigint {
2073
+ try {
2074
+ if (!value || value.length > KNOWLEDGE_BROWSE_CURSOR_MAX_CHARS) {
2075
+ throw new Error("cursor length");
2076
+ }
2077
+ const bytes = Buffer.from(value, "base64url");
2078
+ if (bytes.toString("base64url") !== value) throw new Error("cursor encoding");
2079
+ const parsed = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
2080
+ if (
2081
+ Object.keys(parsed).sort().join(",") !== "q,s,v" ||
2082
+ parsed.v !== 1 ||
2083
+ typeof parsed.s !== "string" ||
2084
+ typeof parsed.q !== "string" ||
2085
+ !/^(0|[1-9][0-9]*)$/.test(parsed.q)
2086
+ ) {
2087
+ throw new Error("cursor payload");
2088
+ }
2089
+ if (parsed.s !== knowledgeBrowseCursorScope(scope)) {
2090
+ throw new Error("knowledge browse cursor belongs to a different scope");
2091
+ }
2092
+ const position = BigInt(parsed.q);
2093
+ if (position > 9_223_372_036_854_775_807n) throw new Error("cursor position");
2094
+ return position;
2095
+ } catch (error) {
2096
+ if (
2097
+ error instanceof Error &&
2098
+ error.message === "knowledge browse cursor belongs to a different scope"
2099
+ ) {
2100
+ throw error;
2101
+ }
2102
+ throw new Error("invalid knowledge browse cursor", { cause: error });
2103
+ }
2104
+ }
2105
+
2106
+ function knowledgeBrowseCursorScope(scope: KnowledgeBrowseCursorScope): string {
2107
+ return createHash("sha256")
2108
+ .update("opengeni:knowledge-browse-cursor:v1\0")
2109
+ .update(scope.accountId)
2110
+ .update("\0")
2111
+ .update(scope.workspaceId)
2112
+ .update("\0")
2113
+ .update(canonicalEffectiveDocumentSubject(scope.initiatingSubjectId))
2114
+ .update("\0")
2115
+ .update(scope.parentId ?? "")
2116
+ .update("\0")
2117
+ .update(scope.topic ?? "")
2118
+ .update("\0")
2119
+ .update([...scope.sourceKinds].sort().join("\0"))
2120
+ .digest("hex");
2121
+ }
2122
+
2123
+ function parseKnowledgeRecordId(value: string): {
2124
+ kind: "document" | "document_chunk";
2125
+ id: string;
2126
+ } {
2127
+ const match =
2128
+ /^(document|document_chunk):([0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/iu.exec(
2129
+ value,
2130
+ );
2131
+ if (!match) throw new Error("invalid knowledge record id");
2132
+ return { kind: match[1] as "document" | "document_chunk", id: match[2]!.toLowerCase() };
2133
+ }
2134
+
2135
+ function knowledgeDocumentRecord(document: typeof schema.documents.$inferSelect): KnowledgeRecord {
2136
+ if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2137
+ const projected = projectKnowledgeRecord({
2138
+ title: document.title,
2139
+ body: document.summary,
2140
+ summary: document.summary,
2141
+ topics: document.topics,
2142
+ metadata: { parser: document.parser, chunkCount: document.chunkCount },
2143
+ source: knowledgeSource(document),
2144
+ });
2145
+ return {
2146
+ id: `document:${document.id}`,
2147
+ kind: "document",
2148
+ title: projected.title,
2149
+ content: projected.content,
2150
+ authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
2151
+ provenance: {
2152
+ source: projected.source,
2153
+ indexedAt: document.indexedAt.toISOString(),
2154
+ },
2155
+ lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2156
+ quality: knowledgeQuality(document),
2157
+ links: knowledgeSourceLinks(projected.source.uri),
2158
+ projection: projected.projection,
2159
+ };
2160
+ }
2161
+
2162
+ function knowledgeChunkRecord(
2163
+ document: typeof schema.documents.$inferSelect,
2164
+ chunk: typeof schema.documentChunks.$inferSelect,
2165
+ ): KnowledgeRecord {
2166
+ if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2167
+ const projected = projectKnowledgeRecord({
2168
+ title: document.title,
2169
+ body: chunk.text,
2170
+ summary: document.summary,
2171
+ topics: document.topics,
2172
+ metadata: { ...chunk.metadata, chunkIndex: chunk.chunkIndex },
2173
+ source: knowledgeSource(document),
2174
+ });
2175
+ return {
2176
+ id: `document_chunk:${chunk.id}`,
2177
+ kind: "document_chunk",
2178
+ title: projected.title,
2179
+ content: projected.content,
2180
+ authority: { kind: normalizeDocumentAuthorityKind(document.authorityKind) },
2181
+ provenance: {
2182
+ source: projected.source,
2183
+ indexedAt: document.indexedAt.toISOString(),
2184
+ },
2185
+ lifecycle: { state: "active", updatedAt: document.updatedAt.toISOString() },
2186
+ quality: knowledgeQuality(document),
2187
+ links: [
2188
+ { relation: "parent", target: { kind: "knowledge", id: `document:${document.id}` } },
2189
+ ...knowledgeSourceLinks(projected.source.uri),
2190
+ ],
2191
+ projection: projected.projection,
2192
+ };
2193
+ }
2194
+
2195
+ function knowledgeSource(document: typeof schema.documents.$inferSelect) {
2196
+ return {
2197
+ kind: normalizeKnowledgeSourceKind(document.sourceKind),
2198
+ uri: document.sourceUri,
2199
+ externalId: document.sourceExternalId,
2200
+ title: document.sourceTitle,
2201
+ author: document.sourceAuthor,
2202
+ createdAt: document.sourceCreatedAt?.toISOString() ?? null,
2203
+ updatedAt: document.sourceUpdatedAt?.toISOString() ?? null,
2204
+ version: document.sourceVersion,
2205
+ };
2206
+ }
2207
+
2208
+ function knowledgeQuality(
2209
+ document: typeof schema.documents.$inferSelect,
2210
+ ): KnowledgeRecord["quality"] {
2211
+ if (!document.indexedAt) throw new Error(`Ready document is missing indexed_at: ${document.id}`);
2212
+ return {
2213
+ trust: "sourced",
2214
+ freshnessAt: (document.sourceUpdatedAt ?? document.indexedAt).toISOString(),
2215
+ conflict: "not_evaluated",
2216
+ correction: "current_source_version",
2217
+ };
2218
+ }
2219
+
2220
+ function knowledgeSourceLinks(sourceUri: string | null): KnowledgeRecord["links"] {
2221
+ return sourceUri ? [{ relation: "source", target: { kind: "external", uri: sourceUri } }] : [];
2222
+ }
2223
+
1796
2224
  async function vectorSearchDocuments(
1797
2225
  db: Database,
1798
2226
  input: DocumentSearchInput,
@@ -0,0 +1,214 @@
1
+ import {
2
+ KNOWLEDGE_BODY_MAX_BYTES,
3
+ KNOWLEDGE_METADATA_MAX_BYTES,
4
+ KNOWLEDGE_METADATA_MAX_DEPTH,
5
+ KNOWLEDGE_METADATA_MAX_ITEMS,
6
+ KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
7
+ KNOWLEDGE_SOURCE_URI_MAX_BYTES,
8
+ KNOWLEDGE_SUMMARY_MAX_BYTES,
9
+ KNOWLEDGE_TITLE_MAX_BYTES,
10
+ KNOWLEDGE_TOPIC_MAX_BYTES,
11
+ KNOWLEDGE_TOPICS_MAX_ITEMS,
12
+ type KnowledgeRecord,
13
+ type KnowledgeSource,
14
+ } from "@opengeni/contracts";
15
+
16
+ type ProjectionField = KnowledgeRecord["projection"]["fields"][number];
17
+
18
+ export type KnowledgeProjectionInput = {
19
+ title: string;
20
+ body: string | null;
21
+ summary: string | null;
22
+ topics: unknown;
23
+ metadata: unknown;
24
+ source: KnowledgeSource;
25
+ };
26
+
27
+ export type KnowledgeProjectionResult = Pick<
28
+ KnowledgeRecord,
29
+ "title" | "content" | "projection"
30
+ > & {
31
+ source: KnowledgeSource;
32
+ };
33
+
34
+ const utf8Bytes = (value: string): number => Buffer.byteLength(value, "utf8");
35
+
36
+ function truncateUtf8(value: string, maxBytes: number): { value: string; truncated: boolean } {
37
+ if (utf8Bytes(value) <= maxBytes) return { value, truncated: false };
38
+ let low = 0;
39
+ let high = value.length;
40
+ while (low < high) {
41
+ const middle = Math.ceil((low + high) / 2);
42
+ const end =
43
+ middle > 0 && middle < value.length && /[\uD800-\uDBFF]/u.test(value[middle - 1]!)
44
+ ? middle - 1
45
+ : middle;
46
+ if (utf8Bytes(value.slice(0, end)) <= maxBytes) low = middle;
47
+ else high = middle - 1;
48
+ }
49
+ let end = low;
50
+ if (end > 0 && end < value.length && /[\uD800-\uDBFF]/u.test(value[end - 1]!)) end -= 1;
51
+ while (end > 0 && utf8Bytes(value.slice(0, end)) > maxBytes) end -= 1;
52
+ return { value: value.slice(0, end), truncated: true };
53
+ }
54
+
55
+ function projectNullableString(
56
+ value: string | null,
57
+ maxBytes: number,
58
+ field: ProjectionField,
59
+ fields: Set<ProjectionField>,
60
+ ): string | null {
61
+ if (value === null) return null;
62
+ const projected = truncateUtf8(value, maxBytes);
63
+ if (projected.truncated) fields.add(field);
64
+ return projected.value;
65
+ }
66
+
67
+ function projectSourceUri(value: string | null, fields: Set<ProjectionField>): string | null {
68
+ if (value === null || value === "") return null;
69
+ if (utf8Bytes(value) > KNOWLEDGE_SOURCE_URI_MAX_BYTES) {
70
+ fields.add("provenance.source.uri");
71
+ return null;
72
+ }
73
+ return value;
74
+ }
75
+
76
+ function projectTopics(value: unknown, fields: Set<ProjectionField>): string[] {
77
+ if (!Array.isArray(value)) {
78
+ if (value !== null && value !== undefined) fields.add("content.topics");
79
+ return [];
80
+ }
81
+ const result: string[] = [];
82
+ for (const candidate of value) {
83
+ if (result.length >= KNOWLEDGE_TOPICS_MAX_ITEMS) {
84
+ fields.add("content.topics");
85
+ break;
86
+ }
87
+ if (typeof candidate !== "string") {
88
+ fields.add("content.topics");
89
+ continue;
90
+ }
91
+ const projected = truncateUtf8(candidate, KNOWLEDGE_TOPIC_MAX_BYTES);
92
+ if (projected.truncated) fields.add("content.topics");
93
+ result.push(projected.value);
94
+ }
95
+ return result;
96
+ }
97
+
98
+ function projectMetadata(value: unknown): { value: Record<string, unknown>; truncated: boolean } {
99
+ let remainingItems = KNOWLEDGE_METADATA_MAX_ITEMS;
100
+ let truncated = false;
101
+
102
+ const visit = (candidate: unknown, depth: number): unknown => {
103
+ if (remainingItems <= 0 || depth > KNOWLEDGE_METADATA_MAX_DEPTH) {
104
+ truncated = true;
105
+ return undefined;
106
+ }
107
+ remainingItems -= 1;
108
+ if (
109
+ candidate === null ||
110
+ typeof candidate === "boolean" ||
111
+ (typeof candidate === "number" && Number.isFinite(candidate))
112
+ ) {
113
+ return candidate;
114
+ }
115
+ if (typeof candidate === "string") {
116
+ const projected = truncateUtf8(candidate, KNOWLEDGE_SOURCE_STRING_MAX_BYTES);
117
+ if (projected.truncated) truncated = true;
118
+ return projected.value;
119
+ }
120
+ if (Array.isArray(candidate)) {
121
+ const result: unknown[] = [];
122
+ for (const item of candidate) {
123
+ const projected = visit(item, depth + 1);
124
+ if (projected === undefined) break;
125
+ result.push(projected);
126
+ }
127
+ return result;
128
+ }
129
+ if (candidate && typeof candidate === "object") {
130
+ const result: Record<string, unknown> = {};
131
+ for (const key of Object.keys(candidate as Record<string, unknown>).sort()) {
132
+ if (utf8Bytes(key) > KNOWLEDGE_TOPIC_MAX_BYTES) {
133
+ truncated = true;
134
+ continue;
135
+ }
136
+ const projected = visit((candidate as Record<string, unknown>)[key], depth + 1);
137
+ if (projected === undefined) break;
138
+ result[key] = projected;
139
+ }
140
+ return result;
141
+ }
142
+ truncated = true;
143
+ return undefined;
144
+ };
145
+
146
+ const root = visit(value, 0);
147
+ let result = root && typeof root === "object" && !Array.isArray(root) ? root : {};
148
+ if (result !== root) truncated = true;
149
+ const bounded: Record<string, unknown> = {};
150
+ for (const key of Object.keys(result as Record<string, unknown>).sort()) {
151
+ bounded[key] = (result as Record<string, unknown>)[key];
152
+ if (utf8Bytes(JSON.stringify(bounded)) > KNOWLEDGE_METADATA_MAX_BYTES) {
153
+ delete bounded[key];
154
+ truncated = true;
155
+ break;
156
+ }
157
+ }
158
+ result = bounded;
159
+ return { value: result as Record<string, unknown>, truncated };
160
+ }
161
+
162
+ /** Deterministic, byte-bounded projection at the agent-facing Knowledge envelope. */
163
+ export function projectKnowledgeRecord(input: KnowledgeProjectionInput): KnowledgeProjectionResult {
164
+ const fields = new Set<ProjectionField>();
165
+ const title = truncateUtf8(input.title, KNOWLEDGE_TITLE_MAX_BYTES);
166
+ if (title.truncated) fields.add("title");
167
+ const metadata = projectMetadata(input.metadata);
168
+ if (metadata.truncated) fields.add("content.metadata");
169
+ const source: KnowledgeSource = {
170
+ ...input.source,
171
+ uri: projectSourceUri(input.source.uri, fields),
172
+ externalId: projectNullableString(
173
+ input.source.externalId,
174
+ KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
175
+ "provenance.source.externalId",
176
+ fields,
177
+ ),
178
+ title: projectNullableString(
179
+ input.source.title,
180
+ KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
181
+ "provenance.source.title",
182
+ fields,
183
+ ),
184
+ author: projectNullableString(
185
+ input.source.author,
186
+ KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
187
+ "provenance.source.author",
188
+ fields,
189
+ ),
190
+ version: projectNullableString(
191
+ input.source.version,
192
+ KNOWLEDGE_SOURCE_STRING_MAX_BYTES,
193
+ "provenance.source.version",
194
+ fields,
195
+ ),
196
+ };
197
+ return {
198
+ title: title.value,
199
+ content: {
200
+ format: "markdown",
201
+ body: projectNullableString(input.body, KNOWLEDGE_BODY_MAX_BYTES, "content.body", fields),
202
+ summary: projectNullableString(
203
+ input.summary,
204
+ KNOWLEDGE_SUMMARY_MAX_BYTES,
205
+ "content.summary",
206
+ fields,
207
+ ),
208
+ topics: projectTopics(input.topics, fields),
209
+ metadata: metadata.value,
210
+ },
211
+ source,
212
+ projection: { truncated: fields.size > 0, fields: [...fields].sort() },
213
+ };
214
+ }