@monoes/memory 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -15,8 +15,14 @@ export interface TextChunk {
|
|
|
15
15
|
/**
|
|
16
16
|
* Chunk a document into overlapping text segments.
|
|
17
17
|
*
|
|
18
|
-
*
|
|
19
|
-
* chunk
|
|
18
|
+
* Boundary preference within 20% of the nominal chunk end:
|
|
19
|
+
* markdown heading (section starts a fresh chunk, no overlap back across it)
|
|
20
|
+
* > paragraph boundary (\n\n) > hard cut.
|
|
21
|
+
*
|
|
22
|
+
* Chunks that don't begin with a heading are prefixed with their governing
|
|
23
|
+
* section heading (`§ <heading>`) so both keyword and semantic retrieval see
|
|
24
|
+
* the section topic — startChar/endChar always refer to the raw source text,
|
|
25
|
+
* never to the prefix.
|
|
20
26
|
*/
|
|
21
27
|
export declare function chunkDocument(docId: string, text: string, chunkSizeChars?: number, overlapChars?: number): TextChunk[];
|
|
22
28
|
//# sourceMappingURL=document-chunker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-chunker.d.ts","sourceRoot":"","sources":["../../src/knowledge/document-chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;
|
|
1
|
+
{"version":3,"file":"document-chunker.d.ts","sourceRoot":"","sources":["../../src/knowledge/document-chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAsBD;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAC3B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAA2B,EAC3C,YAAY,GAAE,MAAwB,GACrC,SAAS,EAAE,CAmEb"}
|
|
@@ -6,11 +6,33 @@
|
|
|
6
6
|
*/
|
|
7
7
|
const DEFAULT_CHUNK_SIZE = 3200;
|
|
8
8
|
const DEFAULT_OVERLAP = 400;
|
|
9
|
+
const HEADING_LINE_RE = /^#{1,6} /;
|
|
10
|
+
/** The markdown heading line governing position `pos`, or null. */
|
|
11
|
+
function lastHeadingBefore(text, pos) {
|
|
12
|
+
let i = text.lastIndexOf('\n#', pos - 1);
|
|
13
|
+
while (i !== -1) {
|
|
14
|
+
const eol = text.indexOf('\n', i + 1);
|
|
15
|
+
const line = text.slice(i + 1, eol === -1 ? undefined : eol);
|
|
16
|
+
if (HEADING_LINE_RE.test(line))
|
|
17
|
+
return line.replace(/^#+ /, '').trim();
|
|
18
|
+
i = text.lastIndexOf('\n#', i - 1);
|
|
19
|
+
}
|
|
20
|
+
const firstEol = text.indexOf('\n');
|
|
21
|
+
const firstLine = firstEol === -1 ? text : text.slice(0, firstEol);
|
|
22
|
+
return HEADING_LINE_RE.test(firstLine) && firstEol !== -1 && firstEol < pos
|
|
23
|
+
? firstLine.replace(/^#+ /, '').trim() : null;
|
|
24
|
+
}
|
|
9
25
|
/**
|
|
10
26
|
* Chunk a document into overlapping text segments.
|
|
11
27
|
*
|
|
12
|
-
*
|
|
13
|
-
* chunk
|
|
28
|
+
* Boundary preference within 20% of the nominal chunk end:
|
|
29
|
+
* markdown heading (section starts a fresh chunk, no overlap back across it)
|
|
30
|
+
* > paragraph boundary (\n\n) > hard cut.
|
|
31
|
+
*
|
|
32
|
+
* Chunks that don't begin with a heading are prefixed with their governing
|
|
33
|
+
* section heading (`§ <heading>`) so both keyword and semantic retrieval see
|
|
34
|
+
* the section topic — startChar/endChar always refer to the raw source text,
|
|
35
|
+
* never to the prefix.
|
|
14
36
|
*/
|
|
15
37
|
export function chunkDocument(docId, text, chunkSizeChars = DEFAULT_CHUNK_SIZE, overlapChars = DEFAULT_OVERLAP) {
|
|
16
38
|
if (text.length === 0) {
|
|
@@ -21,20 +43,40 @@ export function chunkDocument(docId, text, chunkSizeChars = DEFAULT_CHUNK_SIZE,
|
|
|
21
43
|
let chunkIndex = 0;
|
|
22
44
|
while (startChar < text.length) {
|
|
23
45
|
let endChar = Math.min(startChar + chunkSizeChars, text.length);
|
|
24
|
-
|
|
46
|
+
let brokeAtHeading = false;
|
|
25
47
|
if (endChar < text.length) {
|
|
26
48
|
const windowStart = Math.max(startChar, endChar - Math.floor(chunkSizeChars * 0.2));
|
|
27
49
|
const window = text.slice(windowStart, endChar);
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
50
|
+
// Prefer breaking BEFORE a heading so the next section starts fresh
|
|
51
|
+
let h = window.lastIndexOf('\n#');
|
|
52
|
+
while (h !== -1) {
|
|
53
|
+
const eol = window.indexOf('\n', h + 1);
|
|
54
|
+
const line = window.slice(h + 1, eol === -1 ? undefined : eol);
|
|
55
|
+
if (HEADING_LINE_RE.test(line) && windowStart + h > startChar)
|
|
56
|
+
break;
|
|
57
|
+
h = window.lastIndexOf('\n#', h - 1);
|
|
32
58
|
}
|
|
59
|
+
if (h !== -1 && windowStart + h > startChar) {
|
|
60
|
+
endChar = windowStart + h + 1; // keep the \n; heading opens the next chunk
|
|
61
|
+
brokeAtHeading = true;
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
const lastParagraph = window.lastIndexOf('\n\n');
|
|
65
|
+
if (lastParagraph !== -1) {
|
|
66
|
+
// Break right after the paragraph boundary
|
|
67
|
+
endChar = windowStart + lastParagraph + 2;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
let chunkText = text.slice(startChar, endChar);
|
|
72
|
+
const heading = lastHeadingBefore(text, startChar + 1);
|
|
73
|
+
if (heading && !HEADING_LINE_RE.test(chunkText.trimStart())) {
|
|
74
|
+
chunkText = `§ ${heading}\n${chunkText}`;
|
|
33
75
|
}
|
|
34
76
|
chunks.push({
|
|
35
77
|
chunkId: `${docId}:${chunkIndex}`,
|
|
36
78
|
docId,
|
|
37
|
-
text:
|
|
79
|
+
text: chunkText,
|
|
38
80
|
startChar,
|
|
39
81
|
endChar,
|
|
40
82
|
chunkIndex,
|
|
@@ -43,8 +85,12 @@ export function chunkDocument(docId, text, chunkSizeChars = DEFAULT_CHUNK_SIZE,
|
|
|
43
85
|
// If this chunk reached the end of the document, stop
|
|
44
86
|
if (endChar >= text.length)
|
|
45
87
|
break;
|
|
46
|
-
// Advance
|
|
47
|
-
|
|
88
|
+
// Advance with overlap for prose continuity — but never overlap back
|
|
89
|
+
// across a heading break, or the next chunk would re-open the previous
|
|
90
|
+
// section mid-thought.
|
|
91
|
+
const advance = brokeAtHeading
|
|
92
|
+
? Math.max(1, endChar - startChar)
|
|
93
|
+
: Math.max(1, endChar - startChar - overlapChars);
|
|
48
94
|
startChar = startChar + advance;
|
|
49
95
|
}
|
|
50
96
|
return chunks;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-chunker.js","sourceRoot":"","sources":["../../src/knowledge/document-chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B
|
|
1
|
+
{"version":3,"file":"document-chunker.js","sourceRoot":"","sources":["../../src/knowledge/document-chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,MAAM,eAAe,GAAG,UAAU,CAAC;AAEnC,mEAAmE;AACnE,SAAS,iBAAiB,CAAC,IAAY,EAAE,GAAW;IAClD,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7D,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,OAAO,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,GAAG;QACzE,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,KAAa,EACb,IAAY,EACZ,cAAc,GAAW,kBAAkB,EAC3C,YAAY,GAAW,eAAe;IAEtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC,CAAC;YACpF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAEhD,oEAAoE;YACpE,IAAI,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAClC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC/D,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,GAAG,CAAC,GAAG,SAAS;oBAAE,MAAM;gBACrE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;gBAC5C,OAAO,GAAG,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,4CAA4C;gBAC3E,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBACjD,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;oBACzB,2CAA2C;oBAC3C,OAAO,GAAG,WAAW,GAAG,aAAa,GAAG,CAAC,CAAC;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC;YAC5D,SAAS,GAAG,KAAK,OAAO,KAAK,SAAS,EAAE,CAAC;QAC3C,CAAC;QAED,MAAM,CAAC,IAAI,CAAC;YACV,OAAO,EAAE,GAAG,KAAK,IAAI,UAAU,EAAE;YACjC,KAAK;YACL,IAAI,EAAE,SAAS;YACf,SAAS;YACT,OAAO;YACP,UAAU;SACX,CAAC,CAAC;QAEH,UAAU,EAAE,CAAC;QAEb,sDAAsD;QACtD,IAAI,OAAO,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM;QAElC,qEAAqE;QACrE,uEAAuE;QACvE,uBAAuB;QACvB,MAAM,OAAO,GAAG,cAAc;YAC5B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;YAClC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,SAAS,GAAG,YAAY,CAAC,CAAC;QACpD,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;IAClC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@monoes/memory",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Memory module - JSON pattern store, LanceDB + embeddings backend, and a standalone pure-JS HNSW index; three coexisting paths, not a single unified backend",
|
|
6
6
|
"main": "dist/index.js",
|