@remnic/core 9.57.5 → 9.57.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +1 -1
- package/dist/access-authorization-probe.d.ts +1 -1
- package/dist/access-boundary.d.ts +1 -1
- package/dist/access-cli.js +5 -5
- package/dist/access-extraction-force-flush.d.ts +1 -1
- package/dist/access-http-lcm-compaction.d.ts +1 -1
- package/dist/access-http-lifecycle-flush.d.ts +1 -1
- package/dist/access-http-offline-stream.d.ts +1 -1
- package/dist/access-http.d.ts +2 -2
- package/dist/access-lcm-surface.d.ts +1 -1
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-observe-write-surface.d.ts +1 -1
- package/dist/access-operations.d.ts +4 -4
- package/dist/access-recall-concurrency.d.ts +1 -1
- package/dist/access-recall-response.d.ts +1 -1
- package/dist/access-recall-surface.d.ts +1 -1
- package/dist/access-schema.d.ts +88 -88
- package/dist/access-service-helpers.d.ts +1 -1
- package/dist/access-service.d.ts +1 -1
- package/dist/access-surface-catalog.d.ts +1 -1
- package/dist/buffer-surprise.js +3 -3
- package/dist/{chunk-AW4A4XOP.js → chunk-DUHMYBVE.js} +2 -2
- package/dist/chunk-LMXUWAXW.js +200 -0
- package/dist/chunk-LMXUWAXW.js.map +1 -0
- package/dist/{chunk-P4BC54KI.js → chunk-P6WBAR27.js} +8 -32
- package/dist/chunk-P6WBAR27.js.map +1 -0
- package/dist/{chunk-C5B3CIMJ.js → chunk-REVMY7RA.js} +4 -4
- package/dist/{chunk-66SLUXKM.js → chunk-TEHPIMXD.js} +2 -2
- package/dist/chunking.d.ts +3 -1
- package/dist/chunking.js +7 -3
- package/dist/{cli-DTOkYhKI.d.ts → cli-CQdYilcx.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +2 -2
- package/dist/external-wiki-access.d.ts +1 -1
- package/dist/external-wiki-mcp-tools.d.ts +1 -1
- package/dist/index.d.ts +680 -680
- package/dist/index.js +5 -5
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/orchestrator.js +5 -5
- package/dist/{public-http-C01DLhAp.d.ts → public-http-Dw0bJhcC.d.ts} +1 -1
- package/dist/schemas.d.ts +138 -138
- package/dist/semantic-chunking.js +2 -2
- package/dist/shared-context/manager.d.ts +8 -8
- package/dist/support-passport/index.d.ts +4 -4
- package/dist/transfer/types.d.ts +66 -66
- package/package.json +2 -2
- package/src/chunking.ts +119 -34
- package/src/semantic-chunking.ts +10 -48
- package/dist/chunk-P4BC54KI.js.map +0 -1
- package/dist/chunk-UQ7RN5HK.js +0 -126
- package/dist/chunk-UQ7RN5HK.js.map +0 -1
- package/dist/{access-service-D9xXmKvd.d.ts → access-service-4DO8XMq6.d.ts} +64 -64
- /package/dist/{chunk-AW4A4XOP.js.map → chunk-DUHMYBVE.js.map} +0 -0
- /package/dist/{chunk-C5B3CIMJ.js.map → chunk-REVMY7RA.js.map} +0 -0
- /package/dist/{chunk-66SLUXKM.js.map → chunk-TEHPIMXD.js.map} +0 -0
|
@@ -193,7 +193,7 @@ import {
|
|
|
193
193
|
} from "./chunk-NOB2T2FC.js";
|
|
194
194
|
import {
|
|
195
195
|
chunkContent
|
|
196
|
-
} from "./chunk-
|
|
196
|
+
} from "./chunk-LMXUWAXW.js";
|
|
197
197
|
import {
|
|
198
198
|
parseConfig
|
|
199
199
|
} from "./chunk-3VFVQDZO.js";
|
|
@@ -8370,4 +8370,4 @@ export {
|
|
|
8370
8370
|
listMemoryMarkdownFilePaths,
|
|
8371
8371
|
registerCli
|
|
8372
8372
|
};
|
|
8373
|
-
//# sourceMappingURL=chunk-
|
|
8373
|
+
//# sourceMappingURL=chunk-DUHMYBVE.js.map
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// src/chunking.ts
|
|
2
|
+
var DEFAULT_CHUNKING_CONFIG = {
|
|
3
|
+
targetTokens: 200,
|
|
4
|
+
minTokens: 150,
|
|
5
|
+
overlapSentences: 2
|
|
6
|
+
};
|
|
7
|
+
function estimateTokens(text) {
|
|
8
|
+
return Math.ceil(text.length / 4);
|
|
9
|
+
}
|
|
10
|
+
var UNICODE_SENTENCE_TERMINATORS = "\u3002\uFF0E\uFF01\uFF1F\u061F\u06D4\u0964\u0965\uFF61\u2026";
|
|
11
|
+
var ASCII_SENTENCE_TERMINATORS = ".!?";
|
|
12
|
+
var CJK_NO_SPACE_TERMINATORS = "\u3002\uFF0E\uFF01\uFF1F\uFF61";
|
|
13
|
+
var CLOSING_PUNCTUATION = `"'\u201D\u2019\xBB\u300D\u300F\uFF09\uFF3D\u3011\u3009\u300B)]}`;
|
|
14
|
+
var DIGITS = "0123456789\uFF10\uFF11\uFF12\uFF13\uFF14\uFF15\uFF16\uFF17\uFF18\uFF19";
|
|
15
|
+
var SENTENCE_SEGMENTER = typeof Intl.Segmenter === "function" ? new Intl.Segmenter(void 0, { granularity: "sentence" }) : void 0;
|
|
16
|
+
function isSentenceTerminator(character) {
|
|
17
|
+
return ASCII_SENTENCE_TERMINATORS.includes(character) || UNICODE_SENTENCE_TERMINATORS.includes(character);
|
|
18
|
+
}
|
|
19
|
+
function isCjk(character) {
|
|
20
|
+
return /[\u3000-\u9fff\uf900-\ufaff\uac00-\ud7af]/u.test(character);
|
|
21
|
+
}
|
|
22
|
+
function splitSentencesFallback(text) {
|
|
23
|
+
const sentences = [];
|
|
24
|
+
Object.defineProperty(sentences, "separators", {
|
|
25
|
+
value: [],
|
|
26
|
+
writable: true
|
|
27
|
+
});
|
|
28
|
+
let start = 0;
|
|
29
|
+
let separator = "";
|
|
30
|
+
for (let i = 0; i < text.length; i++) {
|
|
31
|
+
const ch = text[i];
|
|
32
|
+
if (!isSentenceTerminator(ch)) continue;
|
|
33
|
+
let end = i;
|
|
34
|
+
let hasUnicodeTerminator = UNICODE_SENTENCE_TERMINATORS.includes(ch);
|
|
35
|
+
while (end + 1 < text.length && isSentenceTerminator(text[end + 1])) {
|
|
36
|
+
end++;
|
|
37
|
+
hasUnicodeTerminator ||= UNICODE_SENTENCE_TERMINATORS.includes(text[end]);
|
|
38
|
+
}
|
|
39
|
+
const isFullwidthDecimal = ch === "\uFF0E" && DIGITS.includes(text[i - 1] ?? "") && DIGITS.includes(text[end + 1] ?? "");
|
|
40
|
+
while (end + 1 < text.length && CLOSING_PUNCTUATION.includes(text[end + 1])) {
|
|
41
|
+
end++;
|
|
42
|
+
}
|
|
43
|
+
const after = text[end + 1];
|
|
44
|
+
const boundary = !isFullwidthDecimal && (hasUnicodeTerminator || after === void 0 || /\s/u.test(after));
|
|
45
|
+
if (boundary) {
|
|
46
|
+
const sentence = text.slice(start, end + 1).trim();
|
|
47
|
+
if (sentence.length > 0) {
|
|
48
|
+
sentences.push(sentence);
|
|
49
|
+
if (sentences.length > 1) sentences.separators.push(separator);
|
|
50
|
+
}
|
|
51
|
+
let nextStart = end + 1;
|
|
52
|
+
while (nextStart < text.length && /\s/u.test(text[nextStart])) nextStart++;
|
|
53
|
+
separator = text.slice(end + 1, nextStart);
|
|
54
|
+
start = nextStart;
|
|
55
|
+
}
|
|
56
|
+
i = end;
|
|
57
|
+
}
|
|
58
|
+
const remaining = text.slice(start).trim();
|
|
59
|
+
if (remaining.length > 0) {
|
|
60
|
+
sentences.push(remaining);
|
|
61
|
+
if (sentences.length > 1) sentences.separators.push(separator);
|
|
62
|
+
}
|
|
63
|
+
return sentences;
|
|
64
|
+
}
|
|
65
|
+
function splitSentences(text) {
|
|
66
|
+
const canUseSegmenter = SENTENCE_SEGMENTER !== void 0 && !/\s/u.test(text) && [...text].some((character) => UNICODE_SENTENCE_TERMINATORS.includes(character));
|
|
67
|
+
if (canUseSegmenter) {
|
|
68
|
+
const segments = Array.from(
|
|
69
|
+
SENTENCE_SEGMENTER.segment(text),
|
|
70
|
+
({ segment }) => segment.trim()
|
|
71
|
+
).filter((segment) => segment.length > 0);
|
|
72
|
+
if (segments.length > 1 && segments.slice(0, -1).every(
|
|
73
|
+
(segment) => [...segment].some(
|
|
74
|
+
(character) => UNICODE_SENTENCE_TERMINATORS.includes(character)
|
|
75
|
+
)
|
|
76
|
+
)) {
|
|
77
|
+
Object.defineProperty(segments, "separators", {
|
|
78
|
+
value: Array(segments.length - 1).fill(""),
|
|
79
|
+
writable: true
|
|
80
|
+
});
|
|
81
|
+
return segments;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return splitSentencesFallback(text);
|
|
85
|
+
}
|
|
86
|
+
function endsWithoutSpace(text) {
|
|
87
|
+
let index = text.length - 1;
|
|
88
|
+
while (index >= 0 && CLOSING_PUNCTUATION.includes(text[index])) index--;
|
|
89
|
+
const terminator = text[index] ?? "";
|
|
90
|
+
if (CJK_NO_SPACE_TERMINATORS.includes(terminator)) {
|
|
91
|
+
return isCjk(text[index - 1] ?? "");
|
|
92
|
+
}
|
|
93
|
+
return terminator === "\u2026" && isCjk(text[index - 1] ?? "");
|
|
94
|
+
}
|
|
95
|
+
function joinSentences(sentences) {
|
|
96
|
+
let result = "";
|
|
97
|
+
const separators = sentences.separators;
|
|
98
|
+
for (let i = 0; i < sentences.length; i++) {
|
|
99
|
+
const sentence = sentences[i];
|
|
100
|
+
if (result.length > 0) {
|
|
101
|
+
const separator = separators?.[i - 1];
|
|
102
|
+
result += separator ?? (endsWithoutSpace(result) ? "" : " ");
|
|
103
|
+
}
|
|
104
|
+
result += sentence;
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
}
|
|
108
|
+
function chunkContent(content, config = DEFAULT_CHUNKING_CONFIG) {
|
|
109
|
+
const totalTokens = estimateTokens(content);
|
|
110
|
+
if (totalTokens < config.minTokens) {
|
|
111
|
+
return {
|
|
112
|
+
chunked: false,
|
|
113
|
+
chunks: [{
|
|
114
|
+
content,
|
|
115
|
+
index: 0,
|
|
116
|
+
tokenCount: totalTokens
|
|
117
|
+
}]
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
const sentences = splitSentences(content);
|
|
121
|
+
if (sentences.length <= 1) {
|
|
122
|
+
return {
|
|
123
|
+
chunked: false,
|
|
124
|
+
chunks: [{
|
|
125
|
+
content,
|
|
126
|
+
index: 0,
|
|
127
|
+
tokenCount: totalTokens
|
|
128
|
+
}]
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
const chunks = [];
|
|
132
|
+
let currentChunkSentences = [];
|
|
133
|
+
let currentTokens = 0;
|
|
134
|
+
let chunkIndex = 0;
|
|
135
|
+
for (let i = 0; i < sentences.length; i++) {
|
|
136
|
+
const sentence = sentences[i];
|
|
137
|
+
const sentenceTokens = estimateTokens(sentence);
|
|
138
|
+
currentChunkSentences.push(sentence);
|
|
139
|
+
currentTokens += sentenceTokens;
|
|
140
|
+
const atTarget = currentTokens >= config.targetTokens;
|
|
141
|
+
const isLastSentence = i === sentences.length - 1;
|
|
142
|
+
if (atTarget || isLastSentence) {
|
|
143
|
+
const chunkContent2 = joinSentences(currentChunkSentences);
|
|
144
|
+
chunks.push({
|
|
145
|
+
content: chunkContent2,
|
|
146
|
+
index: chunkIndex,
|
|
147
|
+
tokenCount: estimateTokens(chunkContent2)
|
|
148
|
+
});
|
|
149
|
+
chunkIndex++;
|
|
150
|
+
if (!isLastSentence) {
|
|
151
|
+
const overlapCount = Math.min(config.overlapSentences, currentChunkSentences.length);
|
|
152
|
+
if (overlapCount <= 0) {
|
|
153
|
+
currentChunkSentences = [];
|
|
154
|
+
currentTokens = 0;
|
|
155
|
+
} else {
|
|
156
|
+
currentChunkSentences = currentChunkSentences.slice(-overlapCount);
|
|
157
|
+
currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
chunked: chunks.length > 1,
|
|
164
|
+
chunks
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function reassembleChunks(chunks) {
|
|
168
|
+
if (chunks.length === 0) return "";
|
|
169
|
+
if (chunks.length === 1) return chunks[0];
|
|
170
|
+
const result = [chunks[0]];
|
|
171
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
172
|
+
const prevChunk = chunks[i - 1];
|
|
173
|
+
const currChunk = chunks[i];
|
|
174
|
+
const prevSentences = splitSentences(prevChunk);
|
|
175
|
+
const currSentences = splitSentences(currChunk);
|
|
176
|
+
let overlapCount = 0;
|
|
177
|
+
for (let j = 0; j < Math.min(prevSentences.length, currSentences.length); j++) {
|
|
178
|
+
const prevEnd = prevSentences.slice(-(j + 1));
|
|
179
|
+
const currStart = currSentences.slice(0, j + 1);
|
|
180
|
+
if (joinSentences(prevEnd) === joinSentences(currStart)) {
|
|
181
|
+
overlapCount = j + 1;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
if (overlapCount > 0 && overlapCount < currSentences.length) {
|
|
185
|
+
result.push(joinSentences(currSentences.slice(overlapCount)));
|
|
186
|
+
} else if (overlapCount === 0) {
|
|
187
|
+
result.push(currChunk);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return joinSentences(result);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export {
|
|
194
|
+
DEFAULT_CHUNKING_CONFIG,
|
|
195
|
+
splitSentences,
|
|
196
|
+
joinSentences,
|
|
197
|
+
chunkContent,
|
|
198
|
+
reassembleChunks
|
|
199
|
+
};
|
|
200
|
+
//# sourceMappingURL=chunk-LMXUWAXW.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/chunking.ts"],"sourcesContent":["/**\n * Automatic Chunking with Overlap (Phase 2A)\n *\n * Sentence-boundary chunking for long memories.\n * Preserves coherent thoughts by never splitting mid-sentence.\n */\n\nexport interface ChunkingConfig {\n /** Target tokens per chunk (default 200) */\n targetTokens: number;\n /** Minimum tokens to trigger chunking (default 150) */\n minTokens: number;\n /** Number of sentences to overlap between chunks (default 2) */\n overlapSentences: number;\n}\n\nexport interface Chunk {\n /** Chunk content */\n content: string;\n /** 0-based index */\n index: number;\n /** Approximate token count */\n tokenCount: number;\n}\n\nexport interface ChunkResult {\n /** Whether content was chunked */\n chunked: boolean;\n /** Array of chunks (length 1 if not chunked) */\n chunks: Chunk[];\n}\n\n/** Default chunking configuration */\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n targetTokens: 200,\n minTokens: 150,\n overlapSentences: 2,\n};\n\n/**\n * Estimate token count for text.\n * Rough approximation: ~4 characters per token for English.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into sentences.\n *\n * The scan stays linear and does not use a backtracking regular expression.\n * ASCII punctuation keeps its old whitespace rule. Unicode terminators also\n * split when the next sentence starts immediately, as in CJK text.\n */\nconst UNICODE_SENTENCE_TERMINATORS = \"。.!?؟۔।॥。…\";\nconst ASCII_SENTENCE_TERMINATORS = \".!?\";\nconst CJK_NO_SPACE_TERMINATORS = \"。.!?。\";\nconst CLOSING_PUNCTUATION = \"\\\"'”’»」』)]】〉》)]}\";\nconst DIGITS = \"01234567890123456789\";\nconst SENTENCE_SEGMENTER =\n typeof Intl.Segmenter === \"function\"\n ? new Intl.Segmenter(undefined, { granularity: \"sentence\" })\n : undefined;\n\nfunction isSentenceTerminator(character: string): boolean {\n return (\n ASCII_SENTENCE_TERMINATORS.includes(character) ||\n UNICODE_SENTENCE_TERMINATORS.includes(character)\n );\n}\n\nfunction isCjk(character: string): boolean {\n return /[\\u3000-\\u9fff\\uf900-\\ufaff\\uac00-\\ud7af]/u.test(character);\n}\n\ntype SentenceList = string[] & { separators: string[] };\n\nfunction splitSentencesFallback(text: string): SentenceList {\n const sentences = [] as unknown as SentenceList;\n Object.defineProperty(sentences, \"separators\", {\n value: [],\n writable: true,\n });\n let start = 0;\n let separator = \"\";\n for (let i = 0; i < text.length; i++) {\n const ch = text[i];\n if (!isSentenceTerminator(ch)) continue;\n\n let end = i;\n let hasUnicodeTerminator = UNICODE_SENTENCE_TERMINATORS.includes(ch);\n while (end + 1 < text.length && isSentenceTerminator(text[end + 1])) {\n end++;\n hasUnicodeTerminator ||= UNICODE_SENTENCE_TERMINATORS.includes(text[end]);\n }\n\n const isFullwidthDecimal =\n ch === \".\" &&\n DIGITS.includes(text[i - 1] ?? \"\") &&\n DIGITS.includes(text[end + 1] ?? \"\");\n while (end + 1 < text.length && CLOSING_PUNCTUATION.includes(text[end + 1])) {\n end++;\n }\n\n const after = text[end + 1];\n const boundary =\n !isFullwidthDecimal &&\n (hasUnicodeTerminator || after === undefined || /\\s/u.test(after));\n if (boundary) {\n const sentence = text.slice(start, end + 1).trim();\n if (sentence.length > 0) {\n sentences.push(sentence);\n if (sentences.length > 1) sentences.separators.push(separator);\n }\n let nextStart = end + 1;\n while (nextStart < text.length && /\\s/u.test(text[nextStart])) nextStart++;\n separator = text.slice(end + 1, nextStart);\n start = nextStart;\n }\n i = end;\n }\n\n const remaining = text.slice(start).trim();\n if (remaining.length > 0) {\n sentences.push(remaining);\n if (sentences.length > 1) sentences.separators.push(separator);\n }\n return sentences;\n}\nexport function splitSentences(text: string): string[] {\n const canUseSegmenter =\n SENTENCE_SEGMENTER !== undefined &&\n !/\\s/u.test(text) &&\n [...text].some((character) => UNICODE_SENTENCE_TERMINATORS.includes(character));\n if (canUseSegmenter) {\n const segments = Array.from(SENTENCE_SEGMENTER.segment(text), ({ segment }) =>\n segment.trim(),\n ).filter((segment) => segment.length > 0) as SentenceList;\n if (\n segments.length > 1 &&\n segments.slice(0, -1).every((segment) =>\n [...segment].some((character) =>\n UNICODE_SENTENCE_TERMINATORS.includes(character),\n ),\n )\n ) {\n Object.defineProperty(segments, \"separators\", {\n value: Array(segments.length - 1).fill(\"\"),\n writable: true,\n });\n return segments;\n }\n }\n return splitSentencesFallback(text);\n}\n\nfunction endsWithoutSpace(text: string): boolean {\n let index = text.length - 1;\n while (index >= 0 && CLOSING_PUNCTUATION.includes(text[index])) index--;\n const terminator = text[index] ?? \"\";\n if (CJK_NO_SPACE_TERMINATORS.includes(terminator)) {\n return isCjk(text[index - 1] ?? \"\");\n }\n return terminator === \"…\" && isCjk(text[index - 1] ?? \"\");\n}\nexport function joinSentences(sentences: string[]): string {\n let result = \"\";\n const separators = (sentences as Partial<SentenceList>).separators;\n for (let i = 0; i < sentences.length; i++) {\n const sentence = sentences[i];\n if (result.length > 0) {\n const separator = separators?.[i - 1];\n result += separator ?? (endsWithoutSpace(result) ? \"\" : \" \");\n }\n result += sentence;\n }\n return result;\n}\n\n\n/**\n * Chunk content into overlapping segments at sentence boundaries.\n *\n * @param content - The text content to chunk\n * @param config - Chunking configuration\n * @returns ChunkResult with chunks array\n */\nexport function chunkContent(\n content: string,\n config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG,\n): ChunkResult {\n const totalTokens = estimateTokens(content);\n\n // Don't chunk if below minimum threshold\n if (totalTokens < config.minTokens) {\n return {\n chunked: false,\n chunks: [{\n content,\n index: 0,\n tokenCount: totalTokens,\n }],\n };\n }\n\n const sentences = splitSentences(content);\n\n // If we couldn't split into multiple sentences, don't chunk\n if (sentences.length <= 1) {\n return {\n chunked: false,\n chunks: [{\n content,\n index: 0,\n tokenCount: totalTokens,\n }],\n };\n }\n\n const chunks: Chunk[] = [];\n let currentChunkSentences: string[] = [];\n let currentTokens = 0;\n let chunkIndex = 0;\n\n for (let i = 0; i < sentences.length; i++) {\n const sentence = sentences[i];\n const sentenceTokens = estimateTokens(sentence);\n\n // Add sentence to current chunk\n currentChunkSentences.push(sentence);\n currentTokens += sentenceTokens;\n\n // Check if we've reached target size (with some flexibility)\n // Allow going over by up to 50% to avoid tiny final chunks\n const atTarget = currentTokens >= config.targetTokens;\n const isLastSentence = i === sentences.length - 1;\n\n if (atTarget || isLastSentence) {\n // Create chunk from accumulated sentences\n const chunkContent = joinSentences(currentChunkSentences);\n chunks.push({\n content: chunkContent,\n index: chunkIndex,\n tokenCount: estimateTokens(chunkContent),\n });\n chunkIndex++;\n\n // Start new chunk with overlap (if not at end)\n if (!isLastSentence) {\n // Keep last N sentences for overlap.\n // Guard: slice(-0) === slice(0), which returns the ENTIRE array\n // (CLAUDE.md gotcha #27). When overlapSentences is 0, clear fully.\n const overlapCount = Math.min(config.overlapSentences, currentChunkSentences.length);\n if (overlapCount <= 0) {\n currentChunkSentences = [];\n currentTokens = 0;\n } else {\n currentChunkSentences = currentChunkSentences.slice(-overlapCount);\n currentTokens = currentChunkSentences.reduce((sum, s) => sum + estimateTokens(s), 0);\n }\n }\n }\n }\n\n // Only consider it \"chunked\" if we got multiple chunks\n return {\n chunked: chunks.length > 1,\n chunks,\n };\n}\n\n/**\n * Get parent content by reassembling chunks.\n * Useful for displaying full context when a chunk is retrieved.\n *\n * @param chunks - Array of chunk contents in order\n * @returns Reassembled parent content (with overlap removed)\n */\nexport function reassembleChunks(chunks: string[]): string {\n if (chunks.length === 0) return \"\";\n if (chunks.length === 1) return chunks[0];\n\n // For overlapping chunks, we need to deduplicate\n // Simple approach: use full first chunk, then non-overlapping parts of subsequent chunks\n // This is imperfect but handles most cases\n const result: string[] = [chunks[0]];\n\n for (let i = 1; i < chunks.length; i++) {\n const prevChunk = chunks[i - 1];\n const currChunk = chunks[i];\n\n // Find overlap by looking for common suffix/prefix\n // Try to find where the previous chunk ends in the current chunk\n const prevSentences = splitSentences(prevChunk);\n const currSentences = splitSentences(currChunk);\n\n // Find how many sentences from prev are at the start of curr\n let overlapCount = 0;\n for (let j = 0; j < Math.min(prevSentences.length, currSentences.length); j++) {\n // Check if last N sentences of prev match first N sentences of curr\n const prevEnd = prevSentences.slice(-(j + 1));\n const currStart = currSentences.slice(0, j + 1);\n\n if (joinSentences(prevEnd) === joinSentences(currStart)) {\n overlapCount = j + 1;\n }\n }\n\n // Add non-overlapping portion\n if (overlapCount > 0 && overlapCount < currSentences.length) {\n result.push(joinSentences(currSentences.slice(overlapCount)));\n } else if (overlapCount === 0) {\n // No detected overlap, add full chunk\n result.push(currChunk);\n }\n // If overlapCount === currSentences.length, skip (fully contained)\n }\n\n return joinSentences(result);\n}\n"],"mappings":";AAiCO,IAAM,0BAA0C;AAAA,EACrD,cAAc;AAAA,EACd,WAAW;AAAA,EACX,kBAAkB;AACpB;AAMA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASA,IAAM,+BAA+B;AACrC,IAAM,6BAA6B;AACnC,IAAM,2BAA2B;AACjC,IAAM,sBAAsB;AAC5B,IAAM,SAAS;AACf,IAAM,qBACJ,OAAO,KAAK,cAAc,aACtB,IAAI,KAAK,UAAU,QAAW,EAAE,aAAa,WAAW,CAAC,IACzD;AAEN,SAAS,qBAAqB,WAA4B;AACxD,SACE,2BAA2B,SAAS,SAAS,KAC7C,6BAA6B,SAAS,SAAS;AAEnD;AAEA,SAAS,MAAM,WAA4B;AACzC,SAAO,6CAA6C,KAAK,SAAS;AACpE;AAIA,SAAS,uBAAuB,MAA4B;AAC1D,QAAM,YAAY,CAAC;AACnB,SAAO,eAAe,WAAW,cAAc;AAAA,IAC7C,OAAO,CAAC;AAAA,IACR,UAAU;AAAA,EACZ,CAAC;AACD,MAAI,QAAQ;AACZ,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,KAAK,KAAK,CAAC;AACjB,QAAI,CAAC,qBAAqB,EAAE,EAAG;AAE/B,QAAI,MAAM;AACV,QAAI,uBAAuB,6BAA6B,SAAS,EAAE;AACnE,WAAO,MAAM,IAAI,KAAK,UAAU,qBAAqB,KAAK,MAAM,CAAC,CAAC,GAAG;AACnE;AACA,+BAAyB,6BAA6B,SAAS,KAAK,GAAG,CAAC;AAAA,IAC1E;AAEA,UAAM,qBACJ,OAAO,YACP,OAAO,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,KACjC,OAAO,SAAS,KAAK,MAAM,CAAC,KAAK,EAAE;AACrC,WAAO,MAAM,IAAI,KAAK,UAAU,oBAAoB,SAAS,KAAK,MAAM,CAAC,CAAC,GAAG;AAC3E;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAM,WACJ,CAAC,uBACA,wBAAwB,UAAU,UAAa,MAAM,KAAK,KAAK;AAClE,QAAI,UAAU;AACZ,YAAM,WAAW,KAAK,MAAM,OAAO,MAAM,CAAC,EAAE,KAAK;AACjD,UAAI,SAAS,SAAS,GAAG;AACvB,kBAAU,KAAK,QAAQ;AACvB,YAAI,UAAU,SAAS,EAAG,WAAU,WAAW,KAAK,SAAS;AAAA,MAC/D;AACA,UAAI,YAAY,MAAM;AACtB,aAAO,YAAY,KAAK,UAAU,MAAM,KAAK,KAAK,SAAS,CAAC,EAAG;AAC/D,kBAAY,KAAK,MAAM,MAAM,GAAG,SAAS;AACzC,cAAQ;AAAA,IACV;AACA,QAAI;AAAA,EACN;AAEA,QAAM,YAAY,KAAK,MAAM,KAAK,EAAE,KAAK;AACzC,MAAI,UAAU,SAAS,GAAG;AACxB,cAAU,KAAK,SAAS;AACxB,QAAI,UAAU,SAAS,EAAG,WAAU,WAAW,KAAK,SAAS;AAAA,EAC/D;AACA,SAAO;AACT;AACO,SAAS,eAAe,MAAwB;AACrD,QAAM,kBACJ,uBAAuB,UACvB,CAAC,MAAM,KAAK,IAAI,KAChB,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,cAAc,6BAA6B,SAAS,SAAS,CAAC;AAChF,MAAI,iBAAiB;AACnB,UAAM,WAAW,MAAM;AAAA,MAAK,mBAAmB,QAAQ,IAAI;AAAA,MAAG,CAAC,EAAE,QAAQ,MACvE,QAAQ,KAAK;AAAA,IACf,EAAE,OAAO,CAAC,YAAY,QAAQ,SAAS,CAAC;AACxC,QACE,SAAS,SAAS,KAClB,SAAS,MAAM,GAAG,EAAE,EAAE;AAAA,MAAM,CAAC,YAC3B,CAAC,GAAG,OAAO,EAAE;AAAA,QAAK,CAAC,cACjB,6BAA6B,SAAS,SAAS;AAAA,MACjD;AAAA,IACF,GACA;AACA,aAAO,eAAe,UAAU,cAAc;AAAA,QAC5C,OAAO,MAAM,SAAS,SAAS,CAAC,EAAE,KAAK,EAAE;AAAA,QACzC,UAAU;AAAA,MACZ,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO,uBAAuB,IAAI;AACpC;AAEA,SAAS,iBAAiB,MAAuB;AAC/C,MAAI,QAAQ,KAAK,SAAS;AAC1B,SAAO,SAAS,KAAK,oBAAoB,SAAS,KAAK,KAAK,CAAC,EAAG;AAChE,QAAM,aAAa,KAAK,KAAK,KAAK;AAClC,MAAI,yBAAyB,SAAS,UAAU,GAAG;AACjD,WAAO,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAE;AAAA,EACpC;AACA,SAAO,eAAe,YAAO,MAAM,KAAK,QAAQ,CAAC,KAAK,EAAE;AAC1D;AACO,SAAS,cAAc,WAA6B;AACzD,MAAI,SAAS;AACb,QAAM,aAAc,UAAoC;AACxD,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,QAAI,OAAO,SAAS,GAAG;AACrB,YAAM,YAAY,aAAa,IAAI,CAAC;AACpC,gBAAU,cAAc,iBAAiB,MAAM,IAAI,KAAK;AAAA,IAC1D;AACA,cAAU;AAAA,EACZ;AACA,SAAO;AACT;AAUO,SAAS,aACd,SACA,SAAyB,yBACZ;AACb,QAAM,cAAc,eAAe,OAAO;AAG1C,MAAI,cAAc,OAAO,WAAW;AAClC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,QACP;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,YAAY,eAAe,OAAO;AAGxC,MAAI,UAAU,UAAU,GAAG;AACzB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,QACP;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,SAAkB,CAAC;AACzB,MAAI,wBAAkC,CAAC;AACvC,MAAI,gBAAgB;AACpB,MAAI,aAAa;AAEjB,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,UAAM,WAAW,UAAU,CAAC;AAC5B,UAAM,iBAAiB,eAAe,QAAQ;AAG9C,0BAAsB,KAAK,QAAQ;AACnC,qBAAiB;AAIjB,UAAM,WAAW,iBAAiB,OAAO;AACzC,UAAM,iBAAiB,MAAM,UAAU,SAAS;AAEhD,QAAI,YAAY,gBAAgB;AAE9B,YAAMA,gBAAe,cAAc,qBAAqB;AACxD,aAAO,KAAK;AAAA,QACV,SAASA;AAAA,QACT,OAAO;AAAA,QACP,YAAY,eAAeA,aAAY;AAAA,MACzC,CAAC;AACD;AAGA,UAAI,CAAC,gBAAgB;AAInB,cAAM,eAAe,KAAK,IAAI,OAAO,kBAAkB,sBAAsB,MAAM;AACnF,YAAI,gBAAgB,GAAG;AACrB,kCAAwB,CAAC;AACzB,0BAAgB;AAAA,QAClB,OAAO;AACL,kCAAwB,sBAAsB,MAAM,CAAC,YAAY;AACjE,0BAAgB,sBAAsB,OAAO,CAAC,KAAK,MAAM,MAAM,eAAe,CAAC,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,SAAO;AAAA,IACL,SAAS,OAAO,SAAS;AAAA,IACzB;AAAA,EACF;AACF;AASO,SAAS,iBAAiB,QAA0B;AACzD,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AAKxC,QAAM,SAAmB,CAAC,OAAO,CAAC,CAAC;AAEnC,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,YAAY,OAAO,IAAI,CAAC;AAC9B,UAAM,YAAY,OAAO,CAAC;AAI1B,UAAM,gBAAgB,eAAe,SAAS;AAC9C,UAAM,gBAAgB,eAAe,SAAS;AAG9C,QAAI,eAAe;AACnB,aAAS,IAAI,GAAG,IAAI,KAAK,IAAI,cAAc,QAAQ,cAAc,MAAM,GAAG,KAAK;AAE7E,YAAM,UAAU,cAAc,MAAM,EAAE,IAAI,EAAE;AAC5C,YAAM,YAAY,cAAc,MAAM,GAAG,IAAI,CAAC;AAE9C,UAAI,cAAc,OAAO,MAAM,cAAc,SAAS,GAAG;AACvD,uBAAe,IAAI;AAAA,MACrB;AAAA,IACF;AAGA,QAAI,eAAe,KAAK,eAAe,cAAc,QAAQ;AAC3D,aAAO,KAAK,cAAc,cAAc,MAAM,YAAY,CAAC,CAAC;AAAA,IAC9D,WAAW,iBAAiB,GAAG;AAE7B,aAAO,KAAK,SAAS;AAAA,IACvB;AAAA,EAEF;AAEA,SAAO,cAAc,MAAM;AAC7B;","names":["chunkContent"]}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
|
-
chunkContent
|
|
3
|
-
|
|
2
|
+
chunkContent,
|
|
3
|
+
joinSentences,
|
|
4
|
+
splitSentences
|
|
5
|
+
} from "./chunk-LMXUWAXW.js";
|
|
4
6
|
|
|
5
7
|
// src/semantic-chunking.ts
|
|
6
8
|
var DEFAULT_SEMANTIC_CHUNKING_CONFIG = {
|
|
@@ -72,32 +74,6 @@ function findLocalMinima(series, threshold) {
|
|
|
72
74
|
}
|
|
73
75
|
return minima;
|
|
74
76
|
}
|
|
75
|
-
function splitSentences(text) {
|
|
76
|
-
const sentences = [];
|
|
77
|
-
let start = 0;
|
|
78
|
-
for (let i = 0; i < text.length; i++) {
|
|
79
|
-
const ch = text[i];
|
|
80
|
-
if (ch !== "." && ch !== "!" && ch !== "?") continue;
|
|
81
|
-
let end = i;
|
|
82
|
-
while (end + 1 < text.length) {
|
|
83
|
-
const n = text[end + 1];
|
|
84
|
-
if (n !== "." && n !== "!" && n !== "?") break;
|
|
85
|
-
end++;
|
|
86
|
-
}
|
|
87
|
-
const after = text[end + 1];
|
|
88
|
-
if (after === void 0 || /\s/.test(after)) {
|
|
89
|
-
const sentence = text.slice(start, end + 1).trim();
|
|
90
|
-
if (sentence.length > 0) sentences.push(sentence);
|
|
91
|
-
start = end + 1;
|
|
92
|
-
}
|
|
93
|
-
i = end;
|
|
94
|
-
}
|
|
95
|
-
if (start < text.length) {
|
|
96
|
-
const remaining = text.slice(start).trim();
|
|
97
|
-
if (remaining.length > 0) sentences.push(remaining);
|
|
98
|
-
}
|
|
99
|
-
return sentences;
|
|
100
|
-
}
|
|
101
77
|
function estimateTokens(text) {
|
|
102
78
|
return Math.ceil(text.length / 4);
|
|
103
79
|
}
|
|
@@ -145,7 +121,7 @@ function mergeShortSegments(segments, minTokens) {
|
|
|
145
121
|
let buffer = [];
|
|
146
122
|
for (let i = 0; i < segments.length; i++) {
|
|
147
123
|
buffer = [...buffer, ...segments[i]];
|
|
148
|
-
const tokenCount = estimateTokens(buffer
|
|
124
|
+
const tokenCount = estimateTokens(joinSentences(buffer));
|
|
149
125
|
if (tokenCount >= minTokens || i === segments.length - 1) {
|
|
150
126
|
merged.push(buffer);
|
|
151
127
|
buffer = [];
|
|
@@ -161,7 +137,7 @@ function mergeShortSegments(segments, minTokens) {
|
|
|
161
137
|
return merged;
|
|
162
138
|
}
|
|
163
139
|
function splitLongSegment(segment, maxTokens, targetTokens) {
|
|
164
|
-
const text = segment
|
|
140
|
+
const text = joinSentences(segment);
|
|
165
141
|
const cappedTarget = Math.min(targetTokens, maxTokens);
|
|
166
142
|
const result = chunkContent(text, {
|
|
167
143
|
targetTokens: cappedTarget,
|
|
@@ -284,7 +260,7 @@ async function semanticChunkContent(content, embedFn, config) {
|
|
|
284
260
|
let sentenceOffset = 0;
|
|
285
261
|
for (let segIdx = 0; segIdx < segments.length; segIdx++) {
|
|
286
262
|
const segment = segments[segIdx];
|
|
287
|
-
const segText = segment
|
|
263
|
+
const segText = joinSentences(segment);
|
|
288
264
|
const segTokens = estimateTokens(segText);
|
|
289
265
|
if (segTokens > cfg.maxTokens) {
|
|
290
266
|
const subChunks = splitLongSegment(segment, cfg.maxTokens, cfg.targetTokens);
|
|
@@ -346,4 +322,4 @@ export {
|
|
|
346
322
|
findLocalMinima,
|
|
347
323
|
semanticChunkContent
|
|
348
324
|
};
|
|
349
|
-
//# sourceMappingURL=chunk-
|
|
325
|
+
//# sourceMappingURL=chunk-P6WBAR27.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/semantic-chunking.ts"],"sourcesContent":["/**\n * Semantic Chunking with Smoothing-Based Topic Boundaries (Issue #368)\n *\n * An optional alternative to the recursive chunker in chunking.ts.\n * Uses sentence embeddings + cosine similarity + smoothing to detect\n * natural topic boundaries, producing more coherent chunks.\n */\n\nimport {\n chunkContent,\n joinSentences,\n splitSentences,\n type Chunk,\n type ChunkResult,\n} from \"./chunking.js\";\n\n// ---------------------------------------------------------------------------\n// Configuration\n// ---------------------------------------------------------------------------\n\nexport interface SemanticChunkingConfig {\n /** Target tokens per chunk. Default: 200. */\n targetTokens: number;\n /** Minimum tokens for a segment before merging with neighbor. Default: 100. */\n minTokens: number;\n /** Maximum tokens for a segment before recursive splitting. Default: 400. */\n maxTokens: number;\n /** Window size for the moving-average smoothing filter. Default: 3. */\n smoothingWindowSize: number;\n /** How many standard deviations below the mean constitutes a boundary. Default: 1.0. */\n boundaryThresholdStdDevs: number;\n /** Batch size for embedding requests. Default: 32. */\n embeddingBatchSize: number;\n /** Fall back to recursive chunking when embeddings are unavailable. Default: true. */\n fallbackToRecursive: boolean;\n}\n\nexport const DEFAULT_SEMANTIC_CHUNKING_CONFIG: SemanticChunkingConfig = {\n targetTokens: 200,\n minTokens: 100,\n maxTokens: 400,\n smoothingWindowSize: 3,\n boundaryThresholdStdDevs: 1.0,\n embeddingBatchSize: 32,\n fallbackToRecursive: true,\n};\n\n// ---------------------------------------------------------------------------\n// Result types\n// ---------------------------------------------------------------------------\n\nexport interface SemanticChunk extends Chunk {\n /** Optional topic hint derived from position. */\n topicLabel?: string;\n /** Cosine similarity score at the trailing boundary of this chunk. */\n boundaryScore: number;\n}\n\nexport interface SemanticChunkResult {\n /** Whether content was split into multiple chunks. */\n chunked: boolean;\n /** The chunks produced. */\n chunks: SemanticChunk[];\n /** Sentence indices where topic splits occurred. */\n boundaries: number[];\n /** Which algorithm produced the result. */\n method: \"semantic\" | \"recursive-fallback\";\n}\n\n// ---------------------------------------------------------------------------\n// Embedding function signature\n// ---------------------------------------------------------------------------\n\n/** Caller-provided function that embeds an array of texts, returning vectors. */\nexport type EmbedFn = (texts: string[]) => Promise<number[][]>;\n\n// ---------------------------------------------------------------------------\n// Math utilities (exported for testing)\n// ---------------------------------------------------------------------------\n\n/**\n * Cosine similarity between two vectors.\n * Returns a value in [-1, 1]. Identical direction = 1, orthogonal = 0.\n *\n * NOTE: This duplicates cosineSimilarity in recall-mmr.ts and embedding-fallback.ts.\n * Consider extracting to a shared math utility in a future refactor.\n */\nexport function cosineSimilarity(a: number[], b: number[]): number {\n if (a.length !== b.length) {\n throw new Error(\n `cosineSimilarity: vector length mismatch (${a.length} vs ${b.length})`,\n );\n }\n if (a.length === 0) return 0;\n\n let dot = 0;\n let magA = 0;\n let magB = 0;\n for (let i = 0; i < a.length; i++) {\n dot += a[i] * b[i];\n magA += a[i] * a[i];\n magB += b[i] * b[i];\n }\n\n const denom = Math.sqrt(magA) * Math.sqrt(magB);\n if (denom === 0) return 0;\n return dot / denom;\n}\n\n/**\n * Arithmetic mean of a numeric series.\n */\nexport function mean(series: number[]): number {\n if (series.length === 0) return 0;\n let sum = 0;\n for (const v of series) sum += v;\n return sum / series.length;\n}\n\n/**\n * Population standard deviation of a numeric series.\n */\nexport function stddev(series: number[]): number {\n if (series.length === 0) return 0;\n const m = mean(series);\n let sumSq = 0;\n for (const v of series) {\n const d = v - m;\n sumSq += d * d;\n }\n return Math.sqrt(sumSq / series.length);\n}\n\n/**\n * Simple moving average over a 1D series.\n * The window is centered: for window size W, each output[i] averages\n * series[i - floor(W/2) .. i + floor(W/2)], clamped to bounds.\n *\n * Even window sizes are rounded up to the next odd value so the window\n * is symmetric around the center point (Finding 4, PR #420).\n */\nexport function movingAverage(series: number[], windowSize: number): number[] {\n if (series.length === 0) return [];\n if (windowSize < 1) windowSize = 1;\n // Round even values up to the next odd so the window is symmetric.\n if (windowSize % 2 === 0) windowSize = windowSize + 1;\n\n const halfW = Math.floor(windowSize / 2);\n const result: number[] = new Array(series.length);\n\n for (let i = 0; i < series.length; i++) {\n const lo = Math.max(0, i - halfW);\n const hi = Math.min(series.length - 1, i + halfW);\n let sum = 0;\n for (let j = lo; j <= hi; j++) sum += series[j];\n result[i] = sum / (hi - lo + 1);\n }\n return result;\n}\n\n/**\n * Find indices in the series that are local minima AND below the threshold.\n * A local minimum is a point lower than both its immediate neighbors\n * (or lower-or-equal at series boundaries).\n */\nexport function findLocalMinima(\n series: number[],\n threshold: number,\n): number[] {\n if (series.length <= 2) return [];\n\n const minima: number[] = [];\n for (let i = 1; i < series.length - 1; i++) {\n if (\n series[i] < series[i - 1] &&\n series[i] < series[i + 1] &&\n series[i] < threshold\n ) {\n minima.push(i);\n }\n }\n return minima;\n}\n\n// ---------------------------------------------------------------------------\n\n// ---------------------------------------------------------------------------\n// Token estimation\n// ---------------------------------------------------------------------------\n\n/** Rough token estimate: ~4 chars per token for English. */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n// ---------------------------------------------------------------------------\n// Core semantic chunking\n// ---------------------------------------------------------------------------\n\n/**\n * Batch-embed sentences using the provided embed function.\n * Respects the configured batch size.\n */\nasync function batchEmbed(\n sentences: string[],\n embedFn: EmbedFn,\n batchSize: number,\n): Promise<number[][]> {\n const allEmbeddings: number[][] = [];\n\n for (let i = 0; i < sentences.length; i += batchSize) {\n const batch = sentences.slice(i, i + batchSize);\n const batchResult = await embedFn(batch);\n for (const vec of batchResult) {\n allEmbeddings.push(vec);\n }\n }\n\n return allEmbeddings;\n}\n\nfunction findEmbeddingDimensionMismatch(\n embeddings: number[][],\n): { expected: number; actual: number; index: number } | null {\n if (embeddings.length <= 1) return null;\n const expected = embeddings[0].length;\n for (let i = 1; i < embeddings.length; i++) {\n const actual = embeddings[i].length;\n if (actual !== expected) {\n return { expected, actual, index: i };\n }\n }\n return null;\n}\n\n/**\n * Build segments from boundary indices.\n * boundaries are sentence indices at which splits occur (i.e., the split\n * happens AFTER the boundary index sentence).\n */\nfunction buildSegments(\n sentences: string[],\n boundaries: number[],\n): string[][] {\n const sorted = [...boundaries].sort((a, b) => a - b);\n const segments: string[][] = [];\n let start = 0;\n\n for (const b of sorted) {\n // Split after sentence at index b: segment is [start .. b]\n const splitPoint = b + 1;\n if (splitPoint > start && splitPoint <= sentences.length) {\n segments.push(sentences.slice(start, splitPoint));\n start = splitPoint;\n }\n }\n\n // Remaining sentences\n if (start < sentences.length) {\n segments.push(sentences.slice(start));\n }\n\n return segments;\n}\n\n/**\n * Merge short segments (below minTokens) with their neighbor.\n * Prefers merging forward; falls back to merging backward.\n */\nfunction mergeShortSegments(\n segments: string[][],\n minTokens: number,\n): string[][] {\n if (segments.length <= 1) return segments;\n\n const merged: string[][] = [];\n let buffer: string[] = [];\n\n for (let i = 0; i < segments.length; i++) {\n buffer = [...buffer, ...segments[i]];\n const tokenCount = estimateTokens(joinSentences(buffer));\n\n if (tokenCount >= minTokens || i === segments.length - 1) {\n merged.push(buffer);\n buffer = [];\n }\n }\n\n // If the last merge left a dangling buffer, attach it to the last segment\n if (buffer.length > 0) {\n if (merged.length > 0) {\n merged[merged.length - 1] = [...merged[merged.length - 1], ...buffer];\n } else {\n merged.push(buffer);\n }\n }\n\n return merged;\n}\n\n/**\n * Split an oversized segment using recursive chunking.\n */\nfunction splitLongSegment(\n segment: string[],\n maxTokens: number,\n targetTokens: number,\n): SemanticChunk[] {\n const text = joinSentences(segment);\n // Cap targetTokens to maxTokens so recursive splitting never produces\n // segments larger than the configured maximum (Finding 2, PR #420).\n const cappedTarget = Math.min(targetTokens, maxTokens);\n const result: ChunkResult = chunkContent(text, {\n targetTokens: cappedTarget,\n minTokens: Math.min(cappedTarget, maxTokens),\n overlapSentences: 0,\n });\n\n return result.chunks.map((c) => ({\n content: c.content,\n index: c.index,\n tokenCount: c.tokenCount,\n boundaryScore: 0,\n }));\n}\n\n/**\n * Semantic chunking with smoothing-based topic boundary detection.\n *\n * @param content - Full text to chunk.\n * @param embedFn - Async function that embeds an array of texts.\n * @param config - Optional partial config overrides.\n * @returns SemanticChunkResult\n */\nexport async function semanticChunkContent(\n content: string,\n embedFn: EmbedFn,\n config?: Partial<SemanticChunkingConfig>,\n): Promise<SemanticChunkResult> {\n const cfg: SemanticChunkingConfig = {\n ...DEFAULT_SEMANTIC_CHUNKING_CONFIG,\n ...config,\n };\n\n // Guard against non-positive batch size which would cause an infinite loop\n const batchSize = Math.max(1, cfg.embeddingBatchSize);\n\n // --- Empty / trivially short input ---\n if (!content || content.trim().length === 0) {\n return {\n chunked: false,\n chunks: [],\n boundaries: [],\n method: \"semantic\",\n };\n }\n\n const sentences = splitSentences(content);\n\n if (sentences.length <= 1) {\n const tokenCount = estimateTokens(content);\n return {\n chunked: false,\n chunks: [\n {\n content: content.trim(),\n index: 0,\n tokenCount,\n boundaryScore: 1,\n },\n ],\n boundaries: [],\n method: \"semantic\",\n };\n }\n\n // If total tokens is short enough, return as single chunk\n const totalTokens = estimateTokens(content);\n if (totalTokens <= cfg.minTokens) {\n return {\n chunked: false,\n chunks: [\n {\n content: content.trim(),\n index: 0,\n tokenCount: totalTokens,\n boundaryScore: 1,\n },\n ],\n boundaries: [],\n method: \"semantic\",\n };\n }\n\n // --- Attempt embedding ---\n let embeddings: number[][];\n try {\n embeddings = await batchEmbed(sentences, embedFn, batchSize);\n } catch {\n // Embedding failed — fall back if configured\n if (cfg.fallbackToRecursive) {\n return buildRecursiveFallback(content, cfg);\n }\n throw new Error(\n \"Semantic chunking failed: embedding function threw and fallbackToRecursive is disabled\",\n );\n }\n\n if (embeddings.length !== sentences.length) {\n if (cfg.fallbackToRecursive) {\n return buildRecursiveFallback(content, cfg);\n }\n throw new Error(\n `Semantic chunking failed: expected ${sentences.length} embeddings but received ${embeddings.length}`,\n );\n }\n\n const dimensionMismatch = findEmbeddingDimensionMismatch(embeddings);\n if (dimensionMismatch) {\n if (cfg.fallbackToRecursive) {\n return buildRecursiveFallback(content, cfg);\n }\n throw new Error(\n `Semantic chunking failed: embedding vectors have mismatched dimensions ` +\n `(${dimensionMismatch.expected} vs ${dimensionMismatch.actual} at index ${dimensionMismatch.index})`,\n );\n }\n\n // --- Compute pairwise cosine similarity ---\n const similarities: number[] = [];\n for (let i = 0; i < sentences.length - 1; i++) {\n similarities.push(cosineSimilarity(embeddings[i], embeddings[i + 1]));\n }\n\n // If only one pair (2 sentences), nothing to smooth or split meaningfully.\n // However, if the combined content exceeds maxTokens, apply recursive splitting.\n if (similarities.length <= 1) {\n if (totalTokens > cfg.maxTokens) {\n return buildRecursiveFallback(content, cfg);\n }\n return {\n chunked: false,\n chunks: [\n {\n content: content.trim(),\n index: 0,\n tokenCount: totalTokens,\n boundaryScore: similarities.length === 1 ? similarities[0] : 1,\n },\n ],\n boundaries: [],\n method: \"semantic\",\n };\n }\n\n // --- Smooth the similarity series ---\n const smoothed = movingAverage(similarities, cfg.smoothingWindowSize);\n\n // --- Detect boundaries: local minima below (mean - k * stddev) ---\n const m = mean(smoothed);\n const s = stddev(smoothed);\n const threshold = m - cfg.boundaryThresholdStdDevs * s;\n const rawBoundaries = findLocalMinima(smoothed, threshold);\n\n // --- Build segments, merge short, split long ---\n let segments = buildSegments(sentences, rawBoundaries);\n segments = mergeShortSegments(segments, cfg.minTokens);\n\n // --- Convert segments to chunks, splitting oversized ones ---\n const chunks: SemanticChunk[] = [];\n const finalBoundaries: number[] = [];\n let sentenceOffset = 0;\n\n for (let segIdx = 0; segIdx < segments.length; segIdx++) {\n const segment = segments[segIdx];\n const segText = joinSentences(segment);\n const segTokens = estimateTokens(segText);\n\n if (segTokens > cfg.maxTokens) {\n // Recursive split for oversized segment\n const subChunks = splitLongSegment(segment, cfg.maxTokens, cfg.targetTokens);\n for (const sc of subChunks) {\n chunks.push({\n ...sc,\n index: chunks.length,\n });\n }\n } else {\n // Compute boundary score: the similarity at the trailing edge\n const trailingSentenceIdx = sentenceOffset + segment.length - 1;\n let bScore = 1;\n if (\n trailingSentenceIdx < similarities.length &&\n segIdx < segments.length - 1\n ) {\n bScore = smoothed[trailingSentenceIdx] ?? similarities[trailingSentenceIdx] ?? 1;\n }\n\n chunks.push({\n content: segText,\n index: chunks.length,\n tokenCount: segTokens,\n boundaryScore: bScore,\n });\n }\n\n // Record boundaries (all but the last segment produce a boundary)\n if (segIdx < segments.length - 1) {\n finalBoundaries.push(sentenceOffset + segment.length - 1);\n }\n sentenceOffset += segment.length;\n }\n\n return {\n chunked: chunks.length > 1,\n chunks,\n boundaries: finalBoundaries,\n method: \"semantic\",\n };\n}\n\n// ---------------------------------------------------------------------------\n// Recursive fallback helper\n// ---------------------------------------------------------------------------\n\nfunction buildRecursiveFallback(\n content: string,\n cfg: SemanticChunkingConfig,\n): SemanticChunkResult {\n // Cap targetTokens to maxTokens so the recursive fallback path honours the\n // same constraint as splitLongSegment (PR #439 post-merge cursor[bot] finding).\n const cappedTarget = Math.min(cfg.targetTokens, cfg.maxTokens);\n const result: ChunkResult = chunkContent(content, {\n targetTokens: cappedTarget,\n minTokens: Math.min(cfg.minTokens, cappedTarget),\n overlapSentences: 0,\n });\n\n return {\n chunked: result.chunked,\n chunks: result.chunks.map((c) => ({\n ...c,\n boundaryScore: 0,\n })),\n boundaries: [],\n method: \"recursive-fallback\",\n };\n}\n"],"mappings":";;;;;;;AAqCO,IAAM,mCAA2D;AAAA,EACtE,cAAc;AAAA,EACd,WAAW;AAAA,EACX,WAAW;AAAA,EACX,qBAAqB;AAAA,EACrB,0BAA0B;AAAA,EAC1B,oBAAoB;AAAA,EACpB,qBAAqB;AACvB;AA0CO,SAAS,iBAAiB,GAAa,GAAqB;AACjE,MAAI,EAAE,WAAW,EAAE,QAAQ;AACzB,UAAM,IAAI;AAAA,MACR,6CAA6C,EAAE,MAAM,OAAO,EAAE,MAAM;AAAA,IACtE;AAAA,EACF;AACA,MAAI,EAAE,WAAW,EAAG,QAAO;AAE3B,MAAI,MAAM;AACV,MAAI,OAAO;AACX,MAAI,OAAO;AACX,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAClB,YAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACpB;AAEA,QAAM,QAAQ,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,IAAI;AAC9C,MAAI,UAAU,EAAG,QAAO;AACxB,SAAO,MAAM;AACf;AAKO,SAAS,KAAK,QAA0B;AAC7C,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,MAAM;AACV,aAAW,KAAK,OAAQ,QAAO;AAC/B,SAAO,MAAM,OAAO;AACtB;AAKO,SAAS,OAAO,QAA0B;AAC/C,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,QAAM,IAAI,KAAK,MAAM;AACrB,MAAI,QAAQ;AACZ,aAAW,KAAK,QAAQ;AACtB,UAAM,IAAI,IAAI;AACd,aAAS,IAAI;AAAA,EACf;AACA,SAAO,KAAK,KAAK,QAAQ,OAAO,MAAM;AACxC;AAUO,SAAS,cAAc,QAAkB,YAA8B;AAC5E,MAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AACjC,MAAI,aAAa,EAAG,cAAa;AAEjC,MAAI,aAAa,MAAM,EAAG,cAAa,aAAa;AAEpD,QAAM,QAAQ,KAAK,MAAM,aAAa,CAAC;AACvC,QAAM,SAAmB,IAAI,MAAM,OAAO,MAAM;AAEhD,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,UAAM,KAAK,KAAK,IAAI,GAAG,IAAI,KAAK;AAChC,UAAM,KAAK,KAAK,IAAI,OAAO,SAAS,GAAG,IAAI,KAAK;AAChD,QAAI,MAAM;AACV,aAAS,IAAI,IAAI,KAAK,IAAI,IAAK,QAAO,OAAO,CAAC;AAC9C,WAAO,CAAC,IAAI,OAAO,KAAK,KAAK;AAAA,EAC/B;AACA,SAAO;AACT;AAOO,SAAS,gBACd,QACA,WACU;AACV,MAAI,OAAO,UAAU,EAAG,QAAO,CAAC;AAEhC,QAAM,SAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,GAAG,KAAK;AAC1C,QACE,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,KACxB,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,KACxB,OAAO,CAAC,IAAI,WACZ;AACA,aAAO,KAAK,CAAC;AAAA,IACf;AAAA,EACF;AACA,SAAO;AACT;AASA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AAUA,eAAe,WACb,WACA,SACA,WACqB;AACrB,QAAM,gBAA4B,CAAC;AAEnC,WAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK,WAAW;AACpD,UAAM,QAAQ,UAAU,MAAM,GAAG,IAAI,SAAS;AAC9C,UAAM,cAAc,MAAM,QAAQ,KAAK;AACvC,eAAW,OAAO,aAAa;AAC7B,oBAAc,KAAK,GAAG;AAAA,IACxB;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,+BACP,YAC4D;AAC5D,MAAI,WAAW,UAAU,EAAG,QAAO;AACnC,QAAM,WAAW,WAAW,CAAC,EAAE;AAC/B,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,UAAM,SAAS,WAAW,CAAC,EAAE;AAC7B,QAAI,WAAW,UAAU;AACvB,aAAO,EAAE,UAAU,QAAQ,OAAO,EAAE;AAAA,IACtC;AAAA,EACF;AACA,SAAO;AACT;AAOA,SAAS,cACP,WACA,YACY;AACZ,QAAM,SAAS,CAAC,GAAG,UAAU,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACnD,QAAM,WAAuB,CAAC;AAC9B,MAAI,QAAQ;AAEZ,aAAW,KAAK,QAAQ;AAEtB,UAAM,aAAa,IAAI;AACvB,QAAI,aAAa,SAAS,cAAc,UAAU,QAAQ;AACxD,eAAS,KAAK,UAAU,MAAM,OAAO,UAAU,CAAC;AAChD,cAAQ;AAAA,IACV;AAAA,EACF;AAGA,MAAI,QAAQ,UAAU,QAAQ;AAC5B,aAAS,KAAK,UAAU,MAAM,KAAK,CAAC;AAAA,EACtC;AAEA,SAAO;AACT;AAMA,SAAS,mBACP,UACA,WACY;AACZ,MAAI,SAAS,UAAU,EAAG,QAAO;AAEjC,QAAM,SAAqB,CAAC;AAC5B,MAAI,SAAmB,CAAC;AAExB,WAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,aAAS,CAAC,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;AACnC,UAAM,aAAa,eAAe,cAAc,MAAM,CAAC;AAEvD,QAAI,cAAc,aAAa,MAAM,SAAS,SAAS,GAAG;AACxD,aAAO,KAAK,MAAM;AAClB,eAAS,CAAC;AAAA,IACZ;AAAA,EACF;AAGA,MAAI,OAAO,SAAS,GAAG;AACrB,QAAI,OAAO,SAAS,GAAG;AACrB,aAAO,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,OAAO,SAAS,CAAC,GAAG,GAAG,MAAM;AAAA,IACtE,OAAO;AACL,aAAO,KAAK,MAAM;AAAA,IACpB;AAAA,EACF;AAEA,SAAO;AACT;AAKA,SAAS,iBACP,SACA,WACA,cACiB;AACjB,QAAM,OAAO,cAAc,OAAO;AAGlC,QAAM,eAAe,KAAK,IAAI,cAAc,SAAS;AACrD,QAAM,SAAsB,aAAa,MAAM;AAAA,IAC7C,cAAc;AAAA,IACd,WAAW,KAAK,IAAI,cAAc,SAAS;AAAA,IAC3C,kBAAkB;AAAA,EACpB,CAAC;AAED,SAAO,OAAO,OAAO,IAAI,CAAC,OAAO;AAAA,IAC/B,SAAS,EAAE;AAAA,IACX,OAAO,EAAE;AAAA,IACT,YAAY,EAAE;AAAA,IACd,eAAe;AAAA,EACjB,EAAE;AACJ;AAUA,eAAsB,qBACpB,SACA,SACA,QAC8B;AAC9B,QAAM,MAA8B;AAAA,IAClC,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAGA,QAAM,YAAY,KAAK,IAAI,GAAG,IAAI,kBAAkB;AAGpD,MAAI,CAAC,WAAW,QAAQ,KAAK,EAAE,WAAW,GAAG;AAC3C,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ,CAAC;AAAA,MACT,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAEA,QAAM,YAAY,eAAe,OAAO;AAExC,MAAI,UAAU,UAAU,GAAG;AACzB,UAAM,aAAa,eAAe,OAAO;AACzC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP;AAAA,UACA,eAAe;AAAA,QACjB;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,cAAc,eAAe,OAAO;AAC1C,MAAI,eAAe,IAAI,WAAW;AAChC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP,YAAY;AAAA,UACZ,eAAe;AAAA,QACjB;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,WAAW,WAAW,SAAS,SAAS;AAAA,EAC7D,QAAQ;AAEN,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI,WAAW,WAAW,UAAU,QAAQ;AAC1C,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR,sCAAsC,UAAU,MAAM,4BAA4B,WAAW,MAAM;AAAA,IACrG;AAAA,EACF;AAEA,QAAM,oBAAoB,+BAA+B,UAAU;AACnE,MAAI,mBAAmB;AACrB,QAAI,IAAI,qBAAqB;AAC3B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,UAAM,IAAI;AAAA,MACR,2EACM,kBAAkB,QAAQ,OAAO,kBAAkB,MAAM,aAAa,kBAAkB,KAAK;AAAA,IACrG;AAAA,EACF;AAGA,QAAM,eAAyB,CAAC;AAChC,WAAS,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,KAAK;AAC7C,iBAAa,KAAK,iBAAiB,WAAW,CAAC,GAAG,WAAW,IAAI,CAAC,CAAC,CAAC;AAAA,EACtE;AAIA,MAAI,aAAa,UAAU,GAAG;AAC5B,QAAI,cAAc,IAAI,WAAW;AAC/B,aAAO,uBAAuB,SAAS,GAAG;AAAA,IAC5C;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,QACN;AAAA,UACE,SAAS,QAAQ,KAAK;AAAA,UACtB,OAAO;AAAA,UACP,YAAY;AAAA,UACZ,eAAe,aAAa,WAAW,IAAI,aAAa,CAAC,IAAI;AAAA,QAC/D;AAAA,MACF;AAAA,MACA,YAAY,CAAC;AAAA,MACb,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,WAAW,cAAc,cAAc,IAAI,mBAAmB;AAGpE,QAAM,IAAI,KAAK,QAAQ;AACvB,QAAM,IAAI,OAAO,QAAQ;AACzB,QAAM,YAAY,IAAI,IAAI,2BAA2B;AACrD,QAAM,gBAAgB,gBAAgB,UAAU,SAAS;AAGzD,MAAI,WAAW,cAAc,WAAW,aAAa;AACrD,aAAW,mBAAmB,UAAU,IAAI,SAAS;AAGrD,QAAM,SAA0B,CAAC;AACjC,QAAM,kBAA4B,CAAC;AACnC,MAAI,iBAAiB;AAErB,WAAS,SAAS,GAAG,SAAS,SAAS,QAAQ,UAAU;AACvD,UAAM,UAAU,SAAS,MAAM;AAC/B,UAAM,UAAU,cAAc,OAAO;AACrC,UAAM,YAAY,eAAe,OAAO;AAExC,QAAI,YAAY,IAAI,WAAW;AAE7B,YAAM,YAAY,iBAAiB,SAAS,IAAI,WAAW,IAAI,YAAY;AAC3E,iBAAW,MAAM,WAAW;AAC1B,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UACH,OAAO,OAAO;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF,OAAO;AAEL,YAAM,sBAAsB,iBAAiB,QAAQ,SAAS;AAC9D,UAAI,SAAS;AACb,UACE,sBAAsB,aAAa,UACnC,SAAS,SAAS,SAAS,GAC3B;AACA,iBAAS,SAAS,mBAAmB,KAAK,aAAa,mBAAmB,KAAK;AAAA,MACjF;AAEA,aAAO,KAAK;AAAA,QACV,SAAS;AAAA,QACT,OAAO,OAAO;AAAA,QACd,YAAY;AAAA,QACZ,eAAe;AAAA,MACjB,CAAC;AAAA,IACH;AAGA,QAAI,SAAS,SAAS,SAAS,GAAG;AAChC,sBAAgB,KAAK,iBAAiB,QAAQ,SAAS,CAAC;AAAA,IAC1D;AACA,sBAAkB,QAAQ;AAAA,EAC5B;AAEA,SAAO;AAAA,IACL,SAAS,OAAO,SAAS;AAAA,IACzB;AAAA,IACA,YAAY;AAAA,IACZ,QAAQ;AAAA,EACV;AACF;AAMA,SAAS,uBACP,SACA,KACqB;AAGrB,QAAM,eAAe,KAAK,IAAI,IAAI,cAAc,IAAI,SAAS;AAC7D,QAAM,SAAsB,aAAa,SAAS;AAAA,IAChD,cAAc;AAAA,IACd,WAAW,KAAK,IAAI,IAAI,WAAW,YAAY;AAAA,IAC/C,kBAAkB;AAAA,EACpB,CAAC;AAED,SAAO;AAAA,IACL,SAAS,OAAO;AAAA,IAChB,QAAQ,OAAO,OAAO,IAAI,CAAC,OAAO;AAAA,MAChC,GAAG;AAAA,MACH,eAAe;AAAA,IACjB,EAAE;AAAA,IACF,YAAY,CAAC;AAAA,IACb,QAAQ;AAAA,EACV;AACF;","names":[]}
|
|
@@ -214,7 +214,7 @@ import {
|
|
|
214
214
|
getWearableConnector,
|
|
215
215
|
listWearableConnectors,
|
|
216
216
|
screenCandidateFact
|
|
217
|
-
} from "./chunk-
|
|
217
|
+
} from "./chunk-DUHMYBVE.js";
|
|
218
218
|
import {
|
|
219
219
|
RoutingRulesStore
|
|
220
220
|
} from "./chunk-TBUEE7XF.js";
|
|
@@ -298,10 +298,10 @@ import {
|
|
|
298
298
|
} from "./chunk-B7RWBZXW.js";
|
|
299
299
|
import {
|
|
300
300
|
semanticChunkContent
|
|
301
|
-
} from "./chunk-
|
|
301
|
+
} from "./chunk-P6WBAR27.js";
|
|
302
302
|
import {
|
|
303
303
|
chunkContent
|
|
304
|
-
} from "./chunk-
|
|
304
|
+
} from "./chunk-LMXUWAXW.js";
|
|
305
305
|
import {
|
|
306
306
|
detectMeetings
|
|
307
307
|
} from "./chunk-3VFVQDZO.js";
|
|
@@ -27127,4 +27127,4 @@ export {
|
|
|
27127
27127
|
blendGraphExpandedRecallScore,
|
|
27128
27128
|
Orchestrator
|
|
27129
27129
|
};
|
|
27130
|
-
//# sourceMappingURL=chunk-
|
|
27130
|
+
//# sourceMappingURL=chunk-REVMY7RA.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
cosineSimilarity
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-P6WBAR27.js";
|
|
4
4
|
|
|
5
5
|
// src/buffer-surprise.ts
|
|
6
6
|
var DEFAULT_SURPRISE_K = 5;
|
|
@@ -64,4 +64,4 @@ export {
|
|
|
64
64
|
DEFAULT_SURPRISE_K,
|
|
65
65
|
computeSurprise
|
|
66
66
|
};
|
|
67
|
-
//# sourceMappingURL=chunk-
|
|
67
|
+
//# sourceMappingURL=chunk-TEHPIMXD.js.map
|
package/dist/chunking.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ interface ChunkResult {
|
|
|
28
28
|
}
|
|
29
29
|
/** Default chunking configuration */
|
|
30
30
|
declare const DEFAULT_CHUNKING_CONFIG: ChunkingConfig;
|
|
31
|
+
declare function splitSentences(text: string): string[];
|
|
32
|
+
declare function joinSentences(sentences: string[]): string;
|
|
31
33
|
/**
|
|
32
34
|
* Chunk content into overlapping segments at sentence boundaries.
|
|
33
35
|
*
|
|
@@ -45,4 +47,4 @@ declare function chunkContent(content: string, config?: ChunkingConfig): ChunkRe
|
|
|
45
47
|
*/
|
|
46
48
|
declare function reassembleChunks(chunks: string[]): string;
|
|
47
49
|
|
|
48
|
-
export { type Chunk, type ChunkResult, type ChunkingConfig, DEFAULT_CHUNKING_CONFIG, chunkContent, reassembleChunks };
|
|
50
|
+
export { type Chunk, type ChunkResult, type ChunkingConfig, DEFAULT_CHUNKING_CONFIG, chunkContent, joinSentences, reassembleChunks, splitSentences };
|
package/dist/chunking.js
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
DEFAULT_CHUNKING_CONFIG,
|
|
3
3
|
chunkContent,
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
joinSentences,
|
|
5
|
+
reassembleChunks,
|
|
6
|
+
splitSentences
|
|
7
|
+
} from "./chunk-LMXUWAXW.js";
|
|
6
8
|
import "./chunk-PZ5AY32C.js";
|
|
7
9
|
export {
|
|
8
10
|
DEFAULT_CHUNKING_CONFIG,
|
|
9
11
|
chunkContent,
|
|
10
|
-
|
|
12
|
+
joinSentences,
|
|
13
|
+
reassembleChunks,
|
|
14
|
+
splitSentences
|
|
11
15
|
};
|
|
12
16
|
//# sourceMappingURL=chunking.js.map
|
|
@@ -5,7 +5,7 @@ import { M as MemoryFile, l as MemoryFrontmatter, y as MemoryStatus, a6 as Behav
|
|
|
5
5
|
import { C as ConversationIndexCoordinator, T as TierMigrationCoordinator, O as Orchestrator } from './orchestrator-DxUWX7gs.js';
|
|
6
6
|
import { S as StorageManager } from './storage-DIyil2ll.js';
|
|
7
7
|
import { TrustZoneDemoSeedResult, TrustZoneName, TrustZonePromotionResult, TrustZoneStoreStatus } from './trust-zones.js';
|
|
8
|
-
import { f as EngramAccessService } from './access-service-
|
|
8
|
+
import { f as EngramAccessService } from './access-service-4DO8XMq6.js';
|
|
9
9
|
import { Writable, Readable } from 'node:stream';
|
|
10
10
|
import { ResolveSecretRefFn } from './resolve-auth-token.js';
|
|
11
11
|
import { CausalTrajectoryStoreStatus } from './causal-trajectory.js';
|
package/dist/cli.d.ts
CHANGED
|
@@ -5,8 +5,8 @@ import './types-CbXieHKr.js';
|
|
|
5
5
|
import './orchestrator-DxUWX7gs.js';
|
|
6
6
|
import './storage-DIyil2ll.js';
|
|
7
7
|
import './trust-zones.js';
|
|
8
|
-
export { c as AccessHttpServeCliCommandOptions, d as ArchiveObservationsCliCommandOptions, e as AuditMemoryCliCommandOptions, B as BulkImportCliCommandOptions, C as CliApi, g as CliCommand, h as CliProgram, i as CompatCliCommandOptions, j as ConnectorsRunCliOutputOptions, k as ConversationIndexHealthCliOrchestrator, D as DashboardStartCliCommandOptions, l as DedupeCandidate, E as ExactDedupePlan, G as GraphHealthCliCommandOptions, M as MemoryActionAuditCliCommandOptions, m as MemoryActionAuditCliNamespaceSummary, n as MemoryActionAuditCliReport, o as MemoryGovernanceCliCommandOptions, q as MemoryGovernanceReportCliCommandOptions, s as MemoryGovernanceRestoreCliCommandOptions, t as MemoryReviewDispositionCliCommandOptions, u as MemoryTimelineCliCommandOptions, v as MigrateCliOrchestrator, w as MigrateCliReport, x as MigrateObservationsCliCommandOptions, P as PolicyDiffCliReport, y as PolicyDiffEntry, z as PolicyRollbackCliReport, F as PolicyStatusCliReport, H as PolicyTuningCliOrchestrator, R as RebuildMemoryLifecycleLedgerCliCommandOptions, I as RebuildMemoryProjectionCliCommandOptions, J as RebuildObservationsCliCommandOptions, K as RepairMemoryProjectionCliCommandOptions, L as ReplayCliCommandOptions, N as ReplayCliOrchestrator, O as RouteCliCommandOptions, S as SessionIntegrityCliCommandOptions, Q as SessionRepairCliCommandOptions, T as TailscaleStatusCliCommandOptions, U as TailscaleSyncCliCommandOptions, V as TierMigrateCliAdapter, W as TierMigrateCoordinatorCliAdapter, X as TierMigrateLegacyCliAdapter, Y as TierStatusCliAdapter, Z as TierStatusCoordinatorCliAdapter, _ as TierStatusLegacyCliAdapter, $ as TrainingExportCliCommandOptions, a0 as VerifyMemoryProjectionCliCommandOptions, a1 as WebDavServeCliCommandOptions, a2 as WorkProjectCliCommandOptions, a3 as WorkTaskCliCommandOptions, a4 as emitConnectorsRunCliResult, a5 as filterNormalMemorySearchResults, a6 as hasDestructivePurgeFailures, a7 as isNormalRetrievalVisibleMemory, a8 as listMemoryMarkdownFilePaths, a9 as parseDurationToMs, aa as parseStrictCliDate, ab as planAggressiveDuplicateDeletions, ac as planExactDuplicateDeletions, ad as registerCli, ae as resolveAccessPrincipalOverride, af as resolveMemoryDirForNamespace, ag as runAbstractionNodeStatusCliCommand, ah as runAccessHttpServeCliCommand, ai as runAccessHttpStatusCliCommand, aj as runAccessHttpStopCliCommand, ak as runAccessMcpServeCliCommand, al as runArchiveObservationsCliCommand, r as runAuditMemoryCliCommand, am as runBenchmarkBaselineReportCliCommand, an as runBenchmarkBaselineSnapshotCliCommand, ao as runBenchmarkCiGateCliCommand, ap as runBenchmarkImportCliCommand, aq as runBenchmarkStatusCliCommand, ar as runBenchmarkStoredBaselineCiGateCliCommand, as as runBenchmarkValidateCliCommand, b as runBulkImportCliCommand, at as runCausalTrajectoryStatusCliCommand, au as runCommitmentLifecycleCliCommand, av as runCommitmentRecordCliCommand, aw as runCommitmentSetStateCliCommand, ax as runCommitmentStatusCliCommand, ay as runCompatCliCommand, az as runCompoundingPromoteCliCommand, aA as runConversationIndexHealthCliCommand, aB as runConversationIndexInspectCliCommand, aC as runConversationIndexRebuildCliCommand, aD as runCueAnchorStatusCliCommand, aE as runDashboardStartCliCommand, aF as runDashboardStatusCliCommand, aG as runDashboardStopCliCommand, aH as runGraphHealthCliCommand, aI as runHarmonicSearchCliCommand, aJ as runMemoryActionAuditCliCommand, aK as runMemoryGovernanceCliCommand, aL as runMemoryGovernanceReportCliCommand, aM as runMemoryGovernanceRestoreCliCommand, aN as runMemoryReviewDispositionCliCommand, aO as runMemoryTimelineCliCommand, aP as runMigrateNormalizeFrontmatterCliCommand, aQ as runMigrateObservationsCliCommand, aR as runMigrateRechunkCliCommand, aS as runMigrateReextractCliCommand, aT as runMigrateRescoreImportanceCliCommand, aU as runObjectiveStateStatusCliCommand, aV as runPolicyDiffCliCommand, aW as runPolicyRollbackCliCommand, aX as runPolicyStatusCliCommand, aY as runRebuildMemoryLifecycleLedgerCliCommand, aZ as runRebuildMemoryProjectionCliCommand, a_ as runRebuildObservationsCliCommand, a$ as runRepairMemoryProjectionCliCommand, b0 as runReplayCliCommand, b1 as runResumeBundleBuildCliCommand, b2 as runResumeBundleRecordCliCommand, b3 as runResumeBundleStatusCliCommand, b4 as runRouteCliCommand, b5 as runSemanticRulePromoteCliCommand, b6 as runSemanticRuleVerifyCliCommand, b7 as runSessionCheckCliCommand, b8 as runSessionRepairCliCommand, b9 as runTailscaleStatusCliCommand, ba as runTailscaleSyncCliCommand, bb as runTierMigrateCliCommand, bc as runTierStatusCliCommand, bd as runTrainingExportCliCommand, be as runTrustZoneDemoSeedCliCommand, bf as runTrustZonePromoteCliCommand, bg as runTrustZoneStatusCliCommand, bh as runUtilityLearningCliCommand, bi as runUtilityLearningStatusCliCommand, bj as runUtilityTelemetryRecordCliCommand, bk as runUtilityTelemetryStatusCliCommand, bl as runVerifiedRecallSearchCliCommand, bm as runVerifyMemoryProjectionCliCommand, bn as runWebDavServeCliCommand, bo as runWebDavStopCliCommand, bp as runWorkProductRecallSearchCliCommand, bq as runWorkProductRecordCliCommand, br as runWorkProductStatusCliCommand, bs as runWorkProjectCliCommand, bt as runWorkTaskCliCommand } from './cli-
|
|
9
|
-
import './access-service-
|
|
8
|
+
export { c as AccessHttpServeCliCommandOptions, d as ArchiveObservationsCliCommandOptions, e as AuditMemoryCliCommandOptions, B as BulkImportCliCommandOptions, C as CliApi, g as CliCommand, h as CliProgram, i as CompatCliCommandOptions, j as ConnectorsRunCliOutputOptions, k as ConversationIndexHealthCliOrchestrator, D as DashboardStartCliCommandOptions, l as DedupeCandidate, E as ExactDedupePlan, G as GraphHealthCliCommandOptions, M as MemoryActionAuditCliCommandOptions, m as MemoryActionAuditCliNamespaceSummary, n as MemoryActionAuditCliReport, o as MemoryGovernanceCliCommandOptions, q as MemoryGovernanceReportCliCommandOptions, s as MemoryGovernanceRestoreCliCommandOptions, t as MemoryReviewDispositionCliCommandOptions, u as MemoryTimelineCliCommandOptions, v as MigrateCliOrchestrator, w as MigrateCliReport, x as MigrateObservationsCliCommandOptions, P as PolicyDiffCliReport, y as PolicyDiffEntry, z as PolicyRollbackCliReport, F as PolicyStatusCliReport, H as PolicyTuningCliOrchestrator, R as RebuildMemoryLifecycleLedgerCliCommandOptions, I as RebuildMemoryProjectionCliCommandOptions, J as RebuildObservationsCliCommandOptions, K as RepairMemoryProjectionCliCommandOptions, L as ReplayCliCommandOptions, N as ReplayCliOrchestrator, O as RouteCliCommandOptions, S as SessionIntegrityCliCommandOptions, Q as SessionRepairCliCommandOptions, T as TailscaleStatusCliCommandOptions, U as TailscaleSyncCliCommandOptions, V as TierMigrateCliAdapter, W as TierMigrateCoordinatorCliAdapter, X as TierMigrateLegacyCliAdapter, Y as TierStatusCliAdapter, Z as TierStatusCoordinatorCliAdapter, _ as TierStatusLegacyCliAdapter, $ as TrainingExportCliCommandOptions, a0 as VerifyMemoryProjectionCliCommandOptions, a1 as WebDavServeCliCommandOptions, a2 as WorkProjectCliCommandOptions, a3 as WorkTaskCliCommandOptions, a4 as emitConnectorsRunCliResult, a5 as filterNormalMemorySearchResults, a6 as hasDestructivePurgeFailures, a7 as isNormalRetrievalVisibleMemory, a8 as listMemoryMarkdownFilePaths, a9 as parseDurationToMs, aa as parseStrictCliDate, ab as planAggressiveDuplicateDeletions, ac as planExactDuplicateDeletions, ad as registerCli, ae as resolveAccessPrincipalOverride, af as resolveMemoryDirForNamespace, ag as runAbstractionNodeStatusCliCommand, ah as runAccessHttpServeCliCommand, ai as runAccessHttpStatusCliCommand, aj as runAccessHttpStopCliCommand, ak as runAccessMcpServeCliCommand, al as runArchiveObservationsCliCommand, r as runAuditMemoryCliCommand, am as runBenchmarkBaselineReportCliCommand, an as runBenchmarkBaselineSnapshotCliCommand, ao as runBenchmarkCiGateCliCommand, ap as runBenchmarkImportCliCommand, aq as runBenchmarkStatusCliCommand, ar as runBenchmarkStoredBaselineCiGateCliCommand, as as runBenchmarkValidateCliCommand, b as runBulkImportCliCommand, at as runCausalTrajectoryStatusCliCommand, au as runCommitmentLifecycleCliCommand, av as runCommitmentRecordCliCommand, aw as runCommitmentSetStateCliCommand, ax as runCommitmentStatusCliCommand, ay as runCompatCliCommand, az as runCompoundingPromoteCliCommand, aA as runConversationIndexHealthCliCommand, aB as runConversationIndexInspectCliCommand, aC as runConversationIndexRebuildCliCommand, aD as runCueAnchorStatusCliCommand, aE as runDashboardStartCliCommand, aF as runDashboardStatusCliCommand, aG as runDashboardStopCliCommand, aH as runGraphHealthCliCommand, aI as runHarmonicSearchCliCommand, aJ as runMemoryActionAuditCliCommand, aK as runMemoryGovernanceCliCommand, aL as runMemoryGovernanceReportCliCommand, aM as runMemoryGovernanceRestoreCliCommand, aN as runMemoryReviewDispositionCliCommand, aO as runMemoryTimelineCliCommand, aP as runMigrateNormalizeFrontmatterCliCommand, aQ as runMigrateObservationsCliCommand, aR as runMigrateRechunkCliCommand, aS as runMigrateReextractCliCommand, aT as runMigrateRescoreImportanceCliCommand, aU as runObjectiveStateStatusCliCommand, aV as runPolicyDiffCliCommand, aW as runPolicyRollbackCliCommand, aX as runPolicyStatusCliCommand, aY as runRebuildMemoryLifecycleLedgerCliCommand, aZ as runRebuildMemoryProjectionCliCommand, a_ as runRebuildObservationsCliCommand, a$ as runRepairMemoryProjectionCliCommand, b0 as runReplayCliCommand, b1 as runResumeBundleBuildCliCommand, b2 as runResumeBundleRecordCliCommand, b3 as runResumeBundleStatusCliCommand, b4 as runRouteCliCommand, b5 as runSemanticRulePromoteCliCommand, b6 as runSemanticRuleVerifyCliCommand, b7 as runSessionCheckCliCommand, b8 as runSessionRepairCliCommand, b9 as runTailscaleStatusCliCommand, ba as runTailscaleSyncCliCommand, bb as runTierMigrateCliCommand, bc as runTierStatusCliCommand, bd as runTrainingExportCliCommand, be as runTrustZoneDemoSeedCliCommand, bf as runTrustZonePromoteCliCommand, bg as runTrustZoneStatusCliCommand, bh as runUtilityLearningCliCommand, bi as runUtilityLearningStatusCliCommand, bj as runUtilityTelemetryRecordCliCommand, bk as runUtilityTelemetryStatusCliCommand, bl as runVerifiedRecallSearchCliCommand, bm as runVerifyMemoryProjectionCliCommand, bn as runWebDavServeCliCommand, bo as runWebDavStopCliCommand, bp as runWorkProductRecallSearchCliCommand, bq as runWorkProductRecordCliCommand, br as runWorkProductStatusCliCommand, bs as runWorkProjectCliCommand, bt as runWorkTaskCliCommand } from './cli-CQdYilcx.js';
|
|
9
|
+
import './access-service-4DO8XMq6.js';
|
|
10
10
|
import 'node:stream';
|
|
11
11
|
import './resolve-auth-token.js';
|
|
12
12
|
import './causal-trajectory.js';
|
package/dist/cli.js
CHANGED
|
@@ -91,7 +91,7 @@ import {
|
|
|
91
91
|
runWorkProductStatusCliCommand,
|
|
92
92
|
runWorkProjectCliCommand,
|
|
93
93
|
runWorkTaskCliCommand
|
|
94
|
-
} from "./chunk-
|
|
94
|
+
} from "./chunk-DUHMYBVE.js";
|
|
95
95
|
import "./chunk-MMACM5G2.js";
|
|
96
96
|
import "./chunk-54BGIBCR.js";
|
|
97
97
|
import "./chunk-H4CKS5B2.js";
|
|
@@ -158,7 +158,7 @@ import "./chunk-2ASAS3CQ.js";
|
|
|
158
158
|
import "./chunk-NOB2T2FC.js";
|
|
159
159
|
import "./chunk-B7RWBZXW.js";
|
|
160
160
|
import "./chunk-YBPYIAA5.js";
|
|
161
|
-
import "./chunk-
|
|
161
|
+
import "./chunk-LMXUWAXW.js";
|
|
162
162
|
import "./chunk-3VFVQDZO.js";
|
|
163
163
|
import "./chunk-HVP33QVW.js";
|
|
164
164
|
import "./chunk-62OKGIJA.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BoundOperation } from './access-boundary.js';
|
|
2
2
|
import { ExternalWikiSearchResult, ExternalWikiSearchInput } from './external-wiki-search.js';
|
|
3
3
|
import 'zod';
|
|
4
|
-
import './access-service-
|
|
4
|
+
import './access-service-4DO8XMq6.js';
|
|
5
5
|
import './catalog-BasnO3Bw.js';
|
|
6
6
|
import './types-CbXieHKr.js';
|
|
7
7
|
import './message-parts/index.js';
|