mcp-rag-memory 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/LICENSE +21 -0
- package/README.md +51 -16
- package/dist/cli.js +85 -32
- package/dist/db/database.d.ts +2 -2
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/database.js +55 -15
- package/dist/db/database.js.map +1 -1
- package/dist/env.js +32 -12
- package/dist/mcp/rag-server.js +20 -12
- package/dist/mcp/rag-server.js.map +1 -1
- package/dist/memory/memory.d.ts +5 -1
- package/dist/memory/memory.d.ts.map +1 -1
- package/dist/memory/memory.js +133 -69
- package/dist/memory/memory.js.map +1 -1
- package/dist/rag/chunker.d.ts.map +1 -1
- package/dist/rag/chunker.js +95 -11
- package/dist/rag/chunker.js.map +1 -1
- package/dist/rag/embeddings.d.ts.map +1 -1
- package/dist/rag/embeddings.js +5 -0
- package/dist/rag/embeddings.js.map +1 -1
- package/dist/rag/pipeline.d.ts +16 -1
- package/dist/rag/pipeline.d.ts.map +1 -1
- package/dist/rag/pipeline.js +122 -31
- package/dist/rag/pipeline.js.map +1 -1
- package/dist/rag/vector-search.d.ts +8 -1
- package/dist/rag/vector-search.d.ts.map +1 -1
- package/dist/rag/vector-search.js +117 -46
- package/dist/rag/vector-search.js.map +1 -1
- package/dist/session/transcript.d.ts +8 -5
- package/dist/session/transcript.d.ts.map +1 -1
- package/dist/session/transcript.js +96 -23
- package/dist/session/transcript.js.map +1 -1
- package/package.json +6 -3
package/dist/rag/chunker.js
CHANGED
|
@@ -18,7 +18,9 @@ function splitIntoParagraphs(text) {
|
|
|
18
18
|
let current = "";
|
|
19
19
|
let inCode = false;
|
|
20
20
|
for (const line of text.split(/\r?\n/)) {
|
|
21
|
-
|
|
21
|
+
// Only real fences toggle code mode — `---` is a thematic break / front
|
|
22
|
+
// matter delimiter, and one stray `---` used to swallow the whole document.
|
|
23
|
+
const codeBoundary = /^\s*(```|~~~)/.test(line.trim());
|
|
22
24
|
if (codeBoundary)
|
|
23
25
|
inCode = !inCode;
|
|
24
26
|
if (inCode || isCodeLine(line)) {
|
|
@@ -38,9 +40,59 @@ function splitIntoParagraphs(text) {
|
|
|
38
40
|
blocks.push(current.trimEnd());
|
|
39
41
|
return blocks.filter((b) => b.trim().length > 0);
|
|
40
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Sentence split that keeps decimals and version numbers intact
|
|
45
|
+
* ("pip 24.0.1" stays one sentence) and keeps fenced code blocks whole,
|
|
46
|
+
* because newlines are never treated as terminators.
|
|
47
|
+
*/
|
|
41
48
|
function splitIntoSentences(block) {
|
|
42
|
-
const
|
|
43
|
-
|
|
49
|
+
const out = [];
|
|
50
|
+
const TERMINATORS = ".!?。！？";
|
|
51
|
+
let start = 0;
|
|
52
|
+
for (let i = 0; i < block.length; i++) {
|
|
53
|
+
const ch = block[i];
|
|
54
|
+
if (!TERMINATORS.includes(ch))
|
|
55
|
+
continue;
|
|
56
|
+
// A period between two digits is a decimal point, not a sentence end.
|
|
57
|
+
if (ch === "." && /\d/.test(block[i - 1] ?? "") && /\d/.test(block[i + 1] ?? ""))
|
|
58
|
+
continue;
|
|
59
|
+
let end = i + 1;
|
|
60
|
+
while (end < block.length && /["')\]}»”]/.test(block[end]))
|
|
61
|
+
end++;
|
|
62
|
+
const piece = block.slice(start, end).trim();
|
|
63
|
+
if (piece)
|
|
64
|
+
out.push(piece);
|
|
65
|
+
start = end;
|
|
66
|
+
i = end - 1;
|
|
67
|
+
}
|
|
68
|
+
const tail = block.slice(start).trim();
|
|
69
|
+
if (tail)
|
|
70
|
+
out.push(tail);
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Trim an overlap tail so a chunk never starts with a partial word: when the cut
|
|
75
|
+
* landed inside a word (last char before the cut and first char of the tail are
|
|
76
|
+
* both word characters), the leading fragment is dropped.
|
|
77
|
+
*/
|
|
78
|
+
function snapToWordStart(tail, charBeforeCut) {
|
|
79
|
+
const at = tail.search(/\S/);
|
|
80
|
+
const body = at <= 0 ? tail : tail.slice(at);
|
|
81
|
+
if (!body)
|
|
82
|
+
return body;
|
|
83
|
+
const startsMidWord = /[\p{L}\p{N}_]/u.test(charBeforeCut) && /[\p{L}\p{N}_]/u.test(body[0]);
|
|
84
|
+
if (!startsMidWord)
|
|
85
|
+
return body;
|
|
86
|
+
const space = body.search(/\s/);
|
|
87
|
+
return space === -1 ? body : body.slice(space).trimStart();
|
|
88
|
+
}
|
|
89
|
+
/** Last whitespace break inside the budget, so a hard cut avoids mid-word cuts. */
|
|
90
|
+
function lastBreak(candidate, max) {
|
|
91
|
+
for (let i = Math.min(candidate.length, max) - 1; i > Math.floor(max * 0.6); i--) {
|
|
92
|
+
if (/\s/.test(candidate[i]))
|
|
93
|
+
return i + 1;
|
|
94
|
+
}
|
|
95
|
+
return max;
|
|
44
96
|
}
|
|
45
97
|
function splitLongText(text, max) {
|
|
46
98
|
const parts = [];
|
|
@@ -48,7 +100,7 @@ function splitLongText(text, max) {
|
|
|
48
100
|
while (rest.length > max) {
|
|
49
101
|
const candidate = rest.slice(0, max);
|
|
50
102
|
const cut = Math.max(candidate.lastIndexOf(". "), candidate.lastIndexOf(".\n"), candidate.lastIndexOf("。"), candidate.lastIndexOf("\n"), candidate.lastIndexOf(", "));
|
|
51
|
-
const pos = cut > Math.floor(max * 0.6) ? cut + 1 : max;
|
|
103
|
+
const pos = cut > Math.floor(max * 0.6) ? cut + 1 : lastBreak(candidate, max);
|
|
52
104
|
parts.push(rest.slice(0, pos).trim());
|
|
53
105
|
rest = rest.slice(pos);
|
|
54
106
|
}
|
|
@@ -56,9 +108,21 @@ function splitLongText(text, max) {
|
|
|
56
108
|
parts.push(rest.trim());
|
|
57
109
|
return parts;
|
|
58
110
|
}
|
|
111
|
+
/** Separator that never mangles code: newline if either side has one. */
|
|
112
|
+
function joiner(left, right) {
|
|
113
|
+
if (!left)
|
|
114
|
+
return "";
|
|
115
|
+
if (left.endsWith("\n") || right.startsWith("\n"))
|
|
116
|
+
return "";
|
|
117
|
+
return left.includes("\n") || right.includes("\n") ? "\n" : " ";
|
|
118
|
+
}
|
|
59
119
|
export function chunkText(text, options = {}) {
|
|
60
120
|
const maxChars = options.maxChars ?? DEFAULT_MAX;
|
|
61
121
|
const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP;
|
|
122
|
+
if (!Number.isFinite(maxChars) || maxChars < 1)
|
|
123
|
+
throw new RangeError("chunkText: maxChars must be >= 1");
|
|
124
|
+
if (!Number.isFinite(overlapChars) || overlapChars < 0)
|
|
125
|
+
throw new RangeError("chunkText: overlapChars must be >= 0");
|
|
62
126
|
const chunks = [];
|
|
63
127
|
let cursor = 0;
|
|
64
128
|
const emit = (content) => {
|
|
@@ -72,9 +136,12 @@ export function chunkText(text, options = {}) {
|
|
|
72
136
|
});
|
|
73
137
|
};
|
|
74
138
|
const paragraphs = splitIntoParagraphs(text);
|
|
139
|
+
const overlap = Math.min(overlapChars, Math.floor(maxChars / 2));
|
|
140
|
+
/** The tail carried into the next chunk; "" when overlapChars is 0. */
|
|
141
|
+
const carry = (emitted) => overlap > 0 ? snapToWordStart(emitted.slice(-overlap), emitted.slice(-overlap - 1, -overlap)) : "";
|
|
75
142
|
let window = "";
|
|
76
143
|
for (const para of paragraphs) {
|
|
77
|
-
let pieces
|
|
144
|
+
let pieces;
|
|
78
145
|
if (para.length > maxChars) {
|
|
79
146
|
pieces = splitLongText(para, maxChars);
|
|
80
147
|
}
|
|
@@ -84,16 +151,33 @@ export function chunkText(text, options = {}) {
|
|
|
84
151
|
}
|
|
85
152
|
for (const piece of pieces) {
|
|
86
153
|
if (piece.length > maxChars) {
|
|
87
|
-
//
|
|
88
|
-
|
|
89
|
-
|
|
154
|
+
// A single sentence longer than maxChars: hard-split it. The pending
|
|
155
|
+
// window must be flushed first, otherwise the text collected so far is
|
|
156
|
+
// emitted twice (once here, once with the next piece appended).
|
|
157
|
+
const hards = splitLongText(piece, maxChars);
|
|
158
|
+
if (window) {
|
|
159
|
+
emit(window);
|
|
160
|
+
window = carry(window);
|
|
161
|
+
}
|
|
162
|
+
for (const hard of hards) {
|
|
163
|
+
const projectedHard = window.length + joiner(window, hard).length + hard.length;
|
|
164
|
+
if (window && projectedHard > maxChars) {
|
|
165
|
+
emit(window);
|
|
166
|
+
window = carry(window);
|
|
167
|
+
}
|
|
168
|
+
window += joiner(window, hard) + hard;
|
|
169
|
+
}
|
|
90
170
|
continue;
|
|
91
171
|
}
|
|
92
|
-
|
|
172
|
+
const projected = window.length + joiner(window, piece).length + piece.length;
|
|
173
|
+
if (window && projected > maxChars) {
|
|
93
174
|
emit(window);
|
|
94
|
-
|
|
175
|
+
// Overlap carries the tail of the previous chunk. overlap === 0 must
|
|
176
|
+
// reset the window: slice(-0) returns the whole string, which used to
|
|
177
|
+
// duplicate the previous chunk and grow it without bound.
|
|
178
|
+
window = carry(window);
|
|
95
179
|
}
|
|
96
|
-
window += (window
|
|
180
|
+
window += joiner(window, piece) + piece;
|
|
97
181
|
}
|
|
98
182
|
}
|
|
99
183
|
if (window.trim())
|
package/dist/rag/chunker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY;YAAE,MAAM,GAAG,CAAC,MAAM,CAAC;QAEnC,IAAI,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,QAAQ,CAAC;IAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,SAAS;QACxC,sEAAsE;QACtE,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,SAAS;QAC3F,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,GAAG,GAAG,CAAC;QACZ,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,aAAqB;IAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;AAC7D,CAAC;AAED,mFAAmF;AACnF,SAAS,SAAS,CAAC,SAAiB,EAAE,GAAW;IAC/C,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjF,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAC5B,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,EAC1B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAC5B,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,SAAS,MAAM,CAAC,IAAY,EAAE,KAAa;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC;IAC7D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IACzG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,sCAAsC,CAAC,CAAC;IACrH,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,EAAE;YACf,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC;SACpC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,uEAAuE;IACvE,MAAM,KAAK,GAAG,CAAC,OAAe,EAAE,EAAE,CAChC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErG,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC5B,qEAAqE;gBACrE,uEAAuE;gBACvE,gEAAgE;gBAChE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,CAAC;oBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzB,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;oBAChF,IAAI,MAAM,IAAI,aAAa,GAAG,QAAQ,EAAE,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBACzB,CAAC;oBACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC;gBACxC,CAAC;gBACD,SAAS;YACX,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC9E,IAAI,MAAM,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,qEAAqE;gBACrE,sEAAsE;gBACtE,0DAA0D;gBAC1D,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,aAAa,IAAI;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM3F;
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,aAAa,IAAI;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM3F;AAyBD,4EAA4E;AAC5E,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAenE"}
|
package/dist/rag/embeddings.js
CHANGED
|
@@ -39,6 +39,8 @@ export function embeddingInfo() {
|
|
|
39
39
|
let extractorPromise = null;
|
|
40
40
|
async function getExtractor() {
|
|
41
41
|
if (!extractorPromise) {
|
|
42
|
+
// A failed model load must not be cached for the process lifetime: reset on
|
|
43
|
+
// rejection so a transient network/first-download error can be retried.
|
|
42
44
|
extractorPromise = (async () => {
|
|
43
45
|
// @ts-expect-error — @huggingface/transformers is an optional dependency
|
|
44
46
|
const mod = await import("@huggingface/transformers");
|
|
@@ -48,6 +50,9 @@ async function getExtractor() {
|
|
|
48
50
|
}
|
|
49
51
|
return pipeline("feature-extraction", EMBEDDING_MODEL, { quantized: true });
|
|
50
52
|
})();
|
|
53
|
+
extractorPromise.catch(() => {
|
|
54
|
+
extractorPromise = null;
|
|
55
|
+
});
|
|
51
56
|
}
|
|
52
57
|
return extractorPromise;
|
|
53
58
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,4EAA4E;QAC5E,wEAAwE;QACxE,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;QACL,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1B,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,IAAI,kBAAkB,KAAK,OAAO,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,IAAyB,CAAC,CAAC;IACzD,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5B,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/rag/pipeline.d.ts
CHANGED
|
@@ -26,6 +26,8 @@ export interface IngestFileOptions {
|
|
|
26
26
|
source?: string;
|
|
27
27
|
contentType?: string;
|
|
28
28
|
metadata?: Record<string, unknown>;
|
|
29
|
+
/** Namespace the dedup hash, so equal text from different scopes is kept apart. */
|
|
30
|
+
dedupScope?: string;
|
|
29
31
|
}
|
|
30
32
|
export interface IngestDirectoryOptions {
|
|
31
33
|
extensions?: string[];
|
|
@@ -37,10 +39,23 @@ export declare function ingestFile(filePath: string, opts?: IngestFileOptions):
|
|
|
37
39
|
export declare function ingestDirectory(dirPath: string, opts?: IngestDirectoryOptions): Promise<{
|
|
38
40
|
ingested: IngestResult[];
|
|
39
41
|
skipped: string[];
|
|
42
|
+
ingestedCount: number;
|
|
43
|
+
skippedCount: number;
|
|
40
44
|
}>;
|
|
41
45
|
export declare function searchDocs(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<SearchHit[]>;
|
|
42
46
|
export declare function retrieve(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<RetrievedContext>;
|
|
43
|
-
|
|
47
|
+
/** Bounded by default: a big store would otherwise dump every row into one tool result. */
|
|
48
|
+
export declare const LIST_DOCS_DEFAULT = 100;
|
|
49
|
+
export declare const LIST_DOCS_MAX = 1000;
|
|
50
|
+
export declare function listDocuments(opts?: {
|
|
51
|
+
limit?: number;
|
|
52
|
+
offset?: number;
|
|
53
|
+
}): {
|
|
54
|
+
documents: Array<Record<string, unknown>>;
|
|
55
|
+
total: number;
|
|
56
|
+
limit: number;
|
|
57
|
+
offset: number;
|
|
58
|
+
};
|
|
44
59
|
export declare function deleteDocument(docId: string): {
|
|
45
60
|
deleted: boolean;
|
|
46
61
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAWD,wBAAsB,UAAU,CAC9B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,YAAY,CAAC,CA+EvB;AA6CD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAuBtG;AAUD,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC;IAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CAgDvG;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,EAAE,QAAQ,SAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG9H;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,SAAI,EACR,QAAQ,SAAO,EACf,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAa3B;AAED,2FAA2F;AAC3F,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,aAAa,OAAO,CAAC;AAElC,wBAAgB,aAAa,CAAC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG;IAC7E,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAeA;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAkBlE;AAED,wBAAgB,aAAa,IAAI;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAMrF;AAYD,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
|
package/dist/rag/pipeline.js
CHANGED
|
@@ -5,7 +5,7 @@ import { chunkText } from "./chunker.js";
|
|
|
5
5
|
import { embedText } from "./embeddings.js";
|
|
6
6
|
import { searchChunks, vectorSearch, invalidateVectorCache } from "./vector-search.js";
|
|
7
7
|
/** Insert a FTS row, tolerating an unavailable FTS5 backend. */
|
|
8
|
-
function insertFtsRow(
|
|
8
|
+
function insertFtsRow(rowid, text) {
|
|
9
9
|
try {
|
|
10
10
|
getDB().prepare("INSERT INTO chunks_fts(rowid, content) VALUES (?, ?)").run(rowid, text);
|
|
11
11
|
}
|
|
@@ -16,8 +16,13 @@ function insertFtsRow(docId, rowid, text) {
|
|
|
16
16
|
export async function ingestText(content, title, opts = {}) {
|
|
17
17
|
const db = getDB();
|
|
18
18
|
const ts = nowMs();
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
if (typeof content !== "string" || content.length === 0) {
|
|
20
|
+
throw new Error("Ingest content must be a non-empty string");
|
|
21
|
+
}
|
|
22
|
+
// Content-hash dedup: identical documents are never re-embedded. `dedupScope`
|
|
23
|
+
// keeps sources that legitimately repeat the same text apart (e.g. "ok" typed
|
|
24
|
+
// in two different sessions).
|
|
25
|
+
const hash = contentHash(`${opts.dedupScope ? `${opts.dedupScope}\u0000` : ""}${content}`);
|
|
21
26
|
const existing = db
|
|
22
27
|
.prepare("SELECT id, title, chunk_count FROM documents WHERE content_hash = ?")
|
|
23
28
|
.get(hash);
|
|
@@ -35,30 +40,76 @@ export async function ingestText(content, title, opts = {}) {
|
|
|
35
40
|
}
|
|
36
41
|
const docId = newId();
|
|
37
42
|
const chunks = chunkText(content);
|
|
38
|
-
db.prepare(`INSERT INTO documents (id, title, source, content_type, metadata, content_hash, chunk_count, created_at, updated_at)
|
|
39
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(docId, title, opts.source ?? null, opts.contentType ?? "text", JSON.stringify(opts.metadata ?? {}), hash, chunks.length, ts, ts);
|
|
40
43
|
let tokens = 0;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
// Embed BEFORE opening the transaction. The MCP SDK does not serialize tool
|
|
45
|
+
// calls, so awaiting inside a transaction let a concurrent request's writes
|
|
46
|
+
// join this one — and a failure here rolled back that other request's work.
|
|
47
|
+
const vectors = [];
|
|
48
|
+
for (const chunk of chunks) {
|
|
49
|
+
vectors.push(await embedText(chunk.text));
|
|
50
|
+
}
|
|
51
|
+
// All writes for one document are atomic, and the block below is fully
|
|
52
|
+
// synchronous, so no other request can interleave inside it.
|
|
53
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
54
|
+
try {
|
|
55
|
+
db.prepare(`INSERT INTO documents (id, title, source, content_type, metadata, content_hash, chunk_count, created_at, updated_at)
|
|
56
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(docId, title, opts.source ?? null, opts.contentType ?? "text", JSON.stringify(opts.metadata ?? {}), hash, chunks.length, ts, ts);
|
|
57
|
+
const insertChunk = db.prepare(`INSERT INTO chunks (id, doc_id, idx, content, embedding, token_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
58
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
59
|
+
const chunk = chunks[i];
|
|
60
|
+
tokens += chunk.tokenCount;
|
|
61
|
+
const res = insertChunk.run(newId(), docId, chunk.index, chunk.text, packVector(vectors[i]), chunk.tokenCount, ts);
|
|
62
|
+
insertFtsRow(Number(res.lastInsertRowid), chunk.text);
|
|
63
|
+
}
|
|
64
|
+
db.exec("COMMIT;");
|
|
65
|
+
}
|
|
66
|
+
catch (e) {
|
|
67
|
+
try {
|
|
68
|
+
db.exec("ROLLBACK;");
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
/* transaction already closed */
|
|
72
|
+
}
|
|
73
|
+
throw e;
|
|
48
74
|
}
|
|
49
75
|
invalidateVectorCache();
|
|
50
76
|
return { docId, title, chunks: chunks.length, tokens, deduplicated: false };
|
|
51
77
|
}
|
|
52
|
-
|
|
78
|
+
/** Roots are realpath'd too, so a junction/symlinked allowlist entry still matches. */
|
|
79
|
+
function allowedRoots() {
|
|
53
80
|
const raw = process.env.RAG_ALLOWED_DIRS;
|
|
54
81
|
if (!raw)
|
|
55
|
-
return;
|
|
56
|
-
|
|
82
|
+
return [];
|
|
83
|
+
return raw
|
|
57
84
|
.split(/[;|,]/)
|
|
58
|
-
.map((p) =>
|
|
59
|
-
.filter(Boolean)
|
|
60
|
-
|
|
85
|
+
.map((p) => p.trim())
|
|
86
|
+
.filter(Boolean)
|
|
87
|
+
.map((p) => {
|
|
88
|
+
const abs = path.resolve(p);
|
|
89
|
+
try {
|
|
90
|
+
return fs.realpathSync(abs);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return abs;
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
let warnedOpenAllowlist = false;
|
|
98
|
+
/**
|
|
99
|
+
* RAG_ALLOWED_DIRS is opt-in, so by default any readable path can be ingested
|
|
100
|
+
* and then stays retrievable from every future session sharing this store. Warn
|
|
101
|
+
* once instead of silently widening the blast radius of an agent-driven tool.
|
|
102
|
+
*/
|
|
103
|
+
function assertAllowedPath(abs) {
|
|
104
|
+
const roots = allowedRoots();
|
|
105
|
+
if (roots.length === 0) {
|
|
106
|
+
if (!warnedOpenAllowlist) {
|
|
107
|
+
warnedOpenAllowlist = true;
|
|
108
|
+
process.stderr.write("rag: RAG_ALLOWED_DIRS is not set — any readable path can be ingested and stays " +
|
|
109
|
+
`retrievable from this store. Set RAG_ALLOWED_DIRS to restrict it.\n`);
|
|
110
|
+
}
|
|
61
111
|
return;
|
|
112
|
+
}
|
|
62
113
|
const real = fs.realpathSync(abs);
|
|
63
114
|
if (!roots.some((r) => real === r || real.startsWith(r + path.sep))) {
|
|
64
115
|
throw new Error(`Path not allowed by RAG_ALLOWED_DIRS: ${abs}`);
|
|
@@ -71,7 +122,8 @@ export async function ingestFile(filePath, opts = {}) {
|
|
|
71
122
|
const st = fs.statSync(abs);
|
|
72
123
|
if (!st.isFile())
|
|
73
124
|
throw new Error(`Not a file: ${abs}`);
|
|
74
|
-
const
|
|
125
|
+
const maxMbRaw = Number(process.env.RAG_MAX_FILE_MB ?? 10);
|
|
126
|
+
const maxMb = Number.isFinite(maxMbRaw) && maxMbRaw > 0 ? maxMbRaw : 10;
|
|
75
127
|
const maxBytes = Math.max(1024, Math.floor(maxMb * 1024 * 1024));
|
|
76
128
|
if (st.size > maxBytes) {
|
|
77
129
|
throw new Error(`File too large (${st.size} bytes > limit ${maxBytes}): ${abs}. Raise RAG_MAX_FILE_MB to allow bigger files.`);
|
|
@@ -89,12 +141,22 @@ export async function ingestFile(filePath, opts = {}) {
|
|
|
89
141
|
function yieldLoop() {
|
|
90
142
|
return new Promise((resolve) => setImmediate(resolve));
|
|
91
143
|
}
|
|
144
|
+
/** Cap the payload: a big directory must not return one line per file. */
|
|
145
|
+
const DIR_REPORT_LIMIT = 50;
|
|
92
146
|
export async function ingestDirectory(dirPath, opts = {}) {
|
|
93
147
|
const root = path.resolve(dirPath);
|
|
94
148
|
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) {
|
|
95
149
|
throw new Error(`Not a directory: ${dirPath}`);
|
|
96
150
|
}
|
|
97
151
|
assertAllowedPath(root);
|
|
152
|
+
if (opts.extensions !== undefined) {
|
|
153
|
+
if (opts.extensions.length === 0)
|
|
154
|
+
throw new Error("extensions must not be empty");
|
|
155
|
+
for (const e of opts.extensions) {
|
|
156
|
+
if (typeof e !== "string" || e.trim() === "")
|
|
157
|
+
throw new Error(`Invalid extension: ${JSON.stringify(e)}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
98
160
|
const exts = opts.extensions?.map((e) => (e.startsWith(".") ? e : `.${e}`)) ?? [
|
|
99
161
|
".ts", ".tsx", ".js", ".jsx", ".json", ".md", ".txt", ".py", ".go", ".rs", ".java", ".c", ".h", ".cpp", ".html", ".css",
|
|
100
162
|
];
|
|
@@ -113,20 +175,28 @@ export async function ingestDirectory(dirPath, opts = {}) {
|
|
|
113
175
|
}
|
|
114
176
|
};
|
|
115
177
|
walk(root);
|
|
178
|
+
const sorted = files.sort();
|
|
116
179
|
const ingested = [];
|
|
117
180
|
const skipped = [];
|
|
118
|
-
|
|
119
|
-
|
|
181
|
+
let ingestedCount = 0;
|
|
182
|
+
let skippedCount = 0;
|
|
183
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
184
|
+
const f = sorted[i];
|
|
120
185
|
try {
|
|
121
|
-
|
|
186
|
+
const res = await ingestFile(f, { metadata: opts.metadata ?? { ingestDir: root } });
|
|
187
|
+
ingestedCount++;
|
|
188
|
+
if (ingested.length < DIR_REPORT_LIMIT)
|
|
189
|
+
ingested.push(res);
|
|
122
190
|
}
|
|
123
191
|
catch (e) {
|
|
124
|
-
|
|
192
|
+
skippedCount++;
|
|
193
|
+
if (skipped.length < DIR_REPORT_LIMIT)
|
|
194
|
+
skipped.push(`${f} (${e.message})`);
|
|
125
195
|
}
|
|
126
196
|
if (i % 10 === 9)
|
|
127
197
|
await yieldLoop();
|
|
128
198
|
}
|
|
129
|
-
return { ingested, skipped };
|
|
199
|
+
return { ingested, skipped, ingestedCount, skippedCount };
|
|
130
200
|
}
|
|
131
201
|
export async function searchDocs(query, topK = 10, minScore = 0.08, filters) {
|
|
132
202
|
const q = await embedText(query);
|
|
@@ -146,19 +216,40 @@ export async function retrieve(query, topK = 6, minScore = 0.08, filters) {
|
|
|
146
216
|
totalTokens: withTokens.reduce((acc, c) => acc + c.tokenCount, 0),
|
|
147
217
|
};
|
|
148
218
|
}
|
|
149
|
-
|
|
219
|
+
/** Bounded by default: a big store would otherwise dump every row into one tool result. */
|
|
220
|
+
export const LIST_DOCS_DEFAULT = 100;
|
|
221
|
+
export const LIST_DOCS_MAX = 1000;
|
|
222
|
+
export function listDocuments(opts = {}) {
|
|
150
223
|
const db = getDB();
|
|
224
|
+
const limit = Math.min(Math.max(Math.trunc(opts.limit ?? LIST_DOCS_DEFAULT), 1), LIST_DOCS_MAX);
|
|
225
|
+
const offset = Math.max(Math.trunc(opts.offset ?? 0), 0);
|
|
226
|
+
const total = db.prepare("SELECT COUNT(*) AS c FROM documents").get().c;
|
|
151
227
|
const rows = db
|
|
152
|
-
.prepare(`SELECT id, title, source, content_type, chunk_count, created_at, updated_at, metadata
|
|
153
|
-
|
|
154
|
-
|
|
228
|
+
.prepare(`SELECT id, title, source, content_type, chunk_count, created_at, updated_at, metadata
|
|
229
|
+
FROM documents ORDER BY updated_at DESC LIMIT ? OFFSET ?`)
|
|
230
|
+
.all(limit, offset);
|
|
231
|
+
return { documents: rows, total, limit, offset };
|
|
155
232
|
}
|
|
156
233
|
export function deleteDocument(docId) {
|
|
157
234
|
const db = getDB();
|
|
158
|
-
db.
|
|
159
|
-
|
|
235
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
236
|
+
let changes = 0;
|
|
237
|
+
try {
|
|
238
|
+
db.prepare("DELETE FROM chunks_fts WHERE rowid IN (SELECT rowid FROM chunks WHERE doc_id = ?)").run(docId);
|
|
239
|
+
changes = Number(db.prepare("DELETE FROM documents WHERE id = ?").run(docId).changes);
|
|
240
|
+
db.exec("COMMIT;");
|
|
241
|
+
}
|
|
242
|
+
catch (e) {
|
|
243
|
+
try {
|
|
244
|
+
db.exec("ROLLBACK;");
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
/* transaction already closed */
|
|
248
|
+
}
|
|
249
|
+
throw e;
|
|
250
|
+
}
|
|
160
251
|
invalidateVectorCache();
|
|
161
|
-
return { deleted:
|
|
252
|
+
return { deleted: changes > 0 };
|
|
162
253
|
}
|
|
163
254
|
export function documentStats() {
|
|
164
255
|
const db = getDB();
|
package/dist/rag/pipeline.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAkCvF,gEAAgE;AAChE,SAAS,YAAY,CAAC,KAAa,EAAE,KAAa,EAAE,IAAY;IAC9D,IAAI,CAAC;QACH,KAAK,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAAe,EACf,KAAa,EACb,OAA0B,EAAE;IAE5B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IAEnB,iEAAiE;IACjE,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,qEAAqE,CAAC;SAC9E,GAAG,CAAC,IAAI,CAAmE,CAAC;IAC/E,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CAAC,wEAAwE,CAAC;aACjF,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAkB,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,EAAE;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,MAAM,CAAC,CAAC;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAElC,EAAE,CAAC,OAAO,CACR;wCACoC,CACrC,CAAC,GAAG,CACH,KAAK,EACL,KAAK,EACL,IAAI,CAAC,MAAM,IAAI,IAAI,EACnB,IAAI,CAAC,WAAW,IAAI,MAAM,EAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,EACnC,IAAI,EACJ,MAAM,CAAC,MAAM,EACb,EAAE,EACF,EAAE,CACH,CAAC;IAEF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAC5B,gHAAgH,CACjH,CAAC;IACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC5G,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IACD,qBAAqB,EAAE,CAAC;IAExB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,KAAK,GAAG,GAAG;SACd,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAClC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC/B,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IACnE,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IAExD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IACjE,IAAI,EAAE,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,mBAAmB,EAAE,CAAC,IAAI,kBAAkB,QAAQ,MAAM,GAAG,gDAAgD,CAC9G,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE;QAChC,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,SAAS,SAAS;IAChB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAExB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI;QAC7E,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;KACxH,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEX,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACzF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAM,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,GAAG,IAAI,EAAE,OAA4B;IACtG,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,YAAY,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,IAAI,GAAG,CAAC,EACR,QAAQ,GAAG,IAAI,EACf,OAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN,+HAA+H,CAChI;SACA,GAAG,EAA+C,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,EAAE,CAAC,OAAO,CAAC,mFAAmF,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3G,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACxE,qBAAqB,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACtF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACrF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAAmB,CAAC;IACzG,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,GAAG,GAA2B;QAClC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;QACvF,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;QAC/F,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;KACjH,CAAC;IACF,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;AAC5B,CAAC;AAED,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAoCvF,gEAAgE;AAChE,SAAS,YAAY,CAAC,KAAa,EAAE,IAAY;IAC/C,IAAI,CAAC;QACH,KAAK,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,gDAAgD;IAClD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAAe,EACf,KAAa,EACb,OAA0B,EAAE;IAE5B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IAED,8EAA8E;IAC9E,8EAA8E;IAC9E,8BAA8B;IAC9B,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,EAAE;SAChB,OAAO,CAAC,qEAAqE,CAAC;SAC9E,GAAG,CAAC,IAAI,CAAmE,CAAC;IAC/E,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CAAC,wEAAwE,CAAC;aACjF,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAkB,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,QAAQ,CAAC,EAAE;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,WAAW;YAC5B,MAAM,EAAE,MAAM,CAAC,CAAC;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,4EAA4E;IAC5E,4EAA4E;IAC5E,4EAA4E;IAC5E,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,uEAAuE;IACvE,6DAA6D;IAC7D,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CACR;0CACoC,CACrC,CAAC,GAAG,CACH,KAAK,EACL,KAAK,EACL,IAAI,CAAC,MAAM,IAAI,IAAI,EACnB,IAAI,CAAC,WAAW,IAAI,MAAM,EAC1B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,EACnC,IAAI,EACJ,MAAM,CAAC,MAAM,EACb,EAAE,EACF,EAAE,CACH,CAAC;QAEF,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAC5B,gHAAgH,CACjH,CAAC;QACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;YAC3B,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACnH,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,qBAAqB,EAAE,CAAC;IAExB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC;AAC9E,CAAC;AAED,uFAAuF;AACvF,SAAS,YAAY;IACnB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,OAAO,GAAG;SACP,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,OAAO,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAED,IAAI,mBAAmB,GAAG,KAAK,CAAC;AAEhC;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACpC,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACzB,mBAAmB,GAAG,IAAI,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iFAAiF;gBAC/E,qEAAqE,CACxE,CAAC;QACJ,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAgB,EAAE,OAA0B,EAAE;IAC7E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;IACnE,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IACjE,IAAI,EAAE,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,mBAAmB,EAAE,CAAC,IAAI,kBAAkB,QAAQ,MAAM,GAAG,gDAAgD,CAC9G,CAAC;IACJ,CAAC;IACD,iBAAiB,CAAC,GAAG,CAAC,CAAC;IAEvB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE;QAChC,MAAM,EAAE,GAAG;QACX,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE;KACzC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,SAAS,SAAS;IAChB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,0EAA0E;AAC1E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IACD,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAExB,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI;QAC7E,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;KACxH,CAAC;IACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,IAAI,CAAC,CAAC;IAEX,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAmB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACpF,aAAa,EAAE,CAAC;YAChB,IAAI,QAAQ,CAAC,MAAM,GAAG,gBAAgB;gBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;YACf,IAAI,OAAO,CAAC,MAAM,GAAG,gBAAgB;gBAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAM,CAAW,CAAC,OAAO,GAAG,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,MAAM,SAAS,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,GAAG,IAAI,EAAE,OAA4B;IACtG,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,YAAY,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,IAAI,GAAG,CAAC,EACR,QAAQ,GAAG,IAAI,EACf,OAA4B;IAE5B,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;KACzB,CAAC,CAAC,CAAC;IACJ,OAAO;QACL,MAAM,EAAE,UAAU;QAClB,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AACrC,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC;AAElC,MAAM,UAAU,aAAa,CAAC,OAA4C,EAAE;IAM1E,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC,EACxD,aAAa,CACd,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAI,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IAC3F,MAAM,IAAI,GAAG,EAAE;SACZ,OAAO,CACN;kEAC4D,CAC7D;SACA,GAAG,CAAC,KAAK,EAAE,MAAM,CAA8C,CAAC;IACnE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CAAC,mFAAmF,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3G,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QACtF,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC;YACH,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,qBAAqB,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACtF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAC;IACrF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,CAAC,CAAC,GAAG,EAAmB,CAAC;IACzG,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7C,MAAM,GAAG,GAA2B;QAClC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;QACvF,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;QAC/F,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;KACjH,CAAC;IACF,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC;AAC5B,CAAC;AAED,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
|
|
@@ -19,6 +19,9 @@ export interface SearchOptions {
|
|
|
19
19
|
topK?: number;
|
|
20
20
|
minScore?: number;
|
|
21
21
|
collection?: "chunks" | "memories";
|
|
22
|
+
onlyKeys?: Set<string>;
|
|
23
|
+
/** Restrict to chunks of these documents (cheap: doc ids, not chunk ids). */
|
|
24
|
+
onlyDocIds?: Set<string>;
|
|
22
25
|
}
|
|
23
26
|
export declare function invalidateVectorCache(): void;
|
|
24
27
|
/** Brute-force vector search over stored vectors (uses the in-memory cache). */
|
|
@@ -37,8 +40,12 @@ export interface SearchChunkOptions {
|
|
|
37
40
|
filters?: SearchChunkFilters;
|
|
38
41
|
}
|
|
39
42
|
/**
|
|
40
|
-
* Hybrid search: FTS5 BM25 keyword + vector, fused with RRF.
|
|
43
|
+
* Hybrid search: FTS5 BM25 keyword + vector, fused with RRF for ranking.
|
|
41
44
|
* Controlled by SEARCH_MODE=hybrid|vector|keyword (default hybrid).
|
|
45
|
+
*
|
|
46
|
+
* The reported `score` is always on a 0..1 relevance scale — the vector cosine
|
|
47
|
+
* when the chunk has one, otherwise a rank-decayed keyword score — so `minScore`
|
|
48
|
+
* means the same thing in every mode (RRF only decides the order).
|
|
42
49
|
*/
|
|
43
50
|
export declare function searchChunks(queryVec: Float64Array, opts: SearchChunkOptions): SearchHit[];
|
|
44
51
|
//# sourceMappingURL=vector-search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../src/rag/vector-search.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../src/rag/vector-search.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,UAAU,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,QAAQ,GAAG,UAAU,CAAC;IACnC,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACvB,6EAA6E;IAC7E,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CAC1B;AAmBD,wBAAgB,qBAAqB,IAAI,IAAI,CAM5C;AAsDD,gFAAgF;AAChF,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,OAAO,GAAE,aAAkB,GAC1B,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAuBvC;AAoED,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,SAAS,EAAE,CAgG1F"}
|