mcp-rag-memory 2.4.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/LICENSE +21 -0
- package/README.md +41 -7
- package/dist/cli.js +85 -32
- package/dist/db/database.d.ts +2 -2
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/database.js +90 -18
- package/dist/db/database.js.map +1 -1
- package/dist/env.js +32 -12
- package/dist/mcp/rag-server.js +20 -12
- package/dist/mcp/rag-server.js.map +1 -1
- package/dist/memory/memory.d.ts +5 -1
- package/dist/memory/memory.d.ts.map +1 -1
- package/dist/memory/memory.js +234 -102
- package/dist/memory/memory.js.map +1 -1
- package/dist/rag/chunker.d.ts +7 -0
- package/dist/rag/chunker.d.ts.map +1 -1
- package/dist/rag/chunker.js +119 -11
- package/dist/rag/chunker.js.map +1 -1
- package/dist/rag/embeddings.d.ts.map +1 -1
- package/dist/rag/embeddings.js +14 -0
- package/dist/rag/embeddings.js.map +1 -1
- package/dist/rag/pipeline.d.ts +18 -1
- package/dist/rag/pipeline.d.ts.map +1 -1
- package/dist/rag/pipeline.js +210 -37
- package/dist/rag/pipeline.js.map +1 -1
- package/dist/rag/vector-search.d.ts +8 -1
- package/dist/rag/vector-search.d.ts.map +1 -1
- package/dist/rag/vector-search.js +126 -51
- package/dist/rag/vector-search.js.map +1 -1
- package/dist/session/transcript.d.ts +8 -5
- package/dist/session/transcript.d.ts.map +1 -1
- package/dist/session/transcript.js +118 -25
- package/dist/session/transcript.js.map +1 -1
- package/package.json +6 -3
package/dist/rag/chunker.js
CHANGED
|
@@ -18,7 +18,9 @@ function splitIntoParagraphs(text) {
|
|
|
18
18
|
let current = "";
|
|
19
19
|
let inCode = false;
|
|
20
20
|
for (const line of text.split(/\r?\n/)) {
|
|
21
|
-
|
|
21
|
+
// Only real fences toggle code mode — `---` is a thematic break / front
|
|
22
|
+
// matter delimiter, and one stray `---` used to swallow the whole document.
|
|
23
|
+
const codeBoundary = /^\s*(```|~~~)/.test(line.trim());
|
|
22
24
|
if (codeBoundary)
|
|
23
25
|
inCode = !inCode;
|
|
24
26
|
if (inCode || isCodeLine(line)) {
|
|
@@ -38,9 +40,59 @@ function splitIntoParagraphs(text) {
|
|
|
38
40
|
blocks.push(current.trimEnd());
|
|
39
41
|
return blocks.filter((b) => b.trim().length > 0);
|
|
40
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Sentence split that keeps decimals and version numbers intact
|
|
45
|
+
* ("pip 24.0.1" stays one sentence) and keeps fenced code blocks whole,
|
|
46
|
+
* because newlines are never treated as terminators.
|
|
47
|
+
*/
|
|
41
48
|
function splitIntoSentences(block) {
|
|
42
|
-
const
|
|
43
|
-
|
|
49
|
+
const out = [];
|
|
50
|
+
const TERMINATORS = ".!?。！？";
|
|
51
|
+
let start = 0;
|
|
52
|
+
for (let i = 0; i < block.length; i++) {
|
|
53
|
+
const ch = block[i];
|
|
54
|
+
if (!TERMINATORS.includes(ch))
|
|
55
|
+
continue;
|
|
56
|
+
// A period between two digits is a decimal point, not a sentence end.
|
|
57
|
+
if (ch === "." && /\d/.test(block[i - 1] ?? "") && /\d/.test(block[i + 1] ?? ""))
|
|
58
|
+
continue;
|
|
59
|
+
let end = i + 1;
|
|
60
|
+
while (end < block.length && /["')\]}»”]/.test(block[end]))
|
|
61
|
+
end++;
|
|
62
|
+
const piece = block.slice(start, end).trim();
|
|
63
|
+
if (piece)
|
|
64
|
+
out.push(piece);
|
|
65
|
+
start = end;
|
|
66
|
+
i = end - 1;
|
|
67
|
+
}
|
|
68
|
+
const tail = block.slice(start).trim();
|
|
69
|
+
if (tail)
|
|
70
|
+
out.push(tail);
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Trim an overlap tail so a chunk never starts with a partial word: when the cut
|
|
75
|
+
* landed inside a word (last char before the cut and first char of the tail are
|
|
76
|
+
* both word characters), the leading fragment is dropped.
|
|
77
|
+
*/
|
|
78
|
+
function snapToWordStart(tail, charBeforeCut) {
|
|
79
|
+
const at = tail.search(/\S/);
|
|
80
|
+
const body = at <= 0 ? tail : tail.slice(at);
|
|
81
|
+
if (!body)
|
|
82
|
+
return body;
|
|
83
|
+
const startsMidWord = /[\p{L}\p{N}_]/u.test(charBeforeCut) && /[\p{L}\p{N}_]/u.test(body[0]);
|
|
84
|
+
if (!startsMidWord)
|
|
85
|
+
return body;
|
|
86
|
+
const space = body.search(/\s/);
|
|
87
|
+
return space === -1 ? body : body.slice(space).trimStart();
|
|
88
|
+
}
|
|
89
|
+
/** Last whitespace break inside the budget, so a hard cut avoids mid-word cuts. */
|
|
90
|
+
function lastBreak(candidate, max) {
|
|
91
|
+
for (let i = Math.min(candidate.length, max) - 1; i > Math.floor(max * 0.6); i--) {
|
|
92
|
+
if (/\s/.test(candidate[i]))
|
|
93
|
+
return i + 1;
|
|
94
|
+
}
|
|
95
|
+
return max;
|
|
44
96
|
}
|
|
45
97
|
function splitLongText(text, max) {
|
|
46
98
|
const parts = [];
|
|
@@ -48,7 +100,7 @@ function splitLongText(text, max) {
|
|
|
48
100
|
while (rest.length > max) {
|
|
49
101
|
const candidate = rest.slice(0, max);
|
|
50
102
|
const cut = Math.max(candidate.lastIndexOf(". "), candidate.lastIndexOf(".\n"), candidate.lastIndexOf("。"), candidate.lastIndexOf("\n"), candidate.lastIndexOf(", "));
|
|
51
|
-
const pos = cut > Math.floor(max * 0.6) ? cut + 1 : max;
|
|
103
|
+
const pos = cut > Math.floor(max * 0.6) ? cut + 1 : lastBreak(candidate, max);
|
|
52
104
|
parts.push(rest.slice(0, pos).trim());
|
|
53
105
|
rest = rest.slice(pos);
|
|
54
106
|
}
|
|
@@ -56,15 +108,40 @@ function splitLongText(text, max) {
|
|
|
56
108
|
parts.push(rest.trim());
|
|
57
109
|
return parts;
|
|
58
110
|
}
|
|
111
|
+
/** Separator that never mangles code: newline if either side has one. */
|
|
112
|
+
function joiner(left, right) {
|
|
113
|
+
if (!left)
|
|
114
|
+
return "";
|
|
115
|
+
if (left.endsWith("\n") || right.startsWith("\n"))
|
|
116
|
+
return "";
|
|
117
|
+
return left.includes("\n") || right.includes("\n") ? "\n" : " ";
|
|
118
|
+
}
|
|
59
119
|
export function chunkText(text, options = {}) {
|
|
60
120
|
const maxChars = options.maxChars ?? DEFAULT_MAX;
|
|
61
121
|
const overlapChars = options.overlapChars ?? DEFAULT_OVERLAP;
|
|
122
|
+
if (!Number.isFinite(maxChars) || maxChars < 1)
|
|
123
|
+
throw new RangeError("chunkText: maxChars must be >= 1");
|
|
124
|
+
if (!Number.isFinite(overlapChars) || overlapChars < 0)
|
|
125
|
+
throw new RangeError("chunkText: overlapChars must be >= 0");
|
|
62
126
|
const chunks = [];
|
|
63
127
|
let cursor = 0;
|
|
64
128
|
const emit = (content) => {
|
|
65
129
|
const trimmed = content.trim();
|
|
66
130
|
if (!trimmed)
|
|
67
131
|
return;
|
|
132
|
+
// Backstop for maxChars. The window is normally kept under budget, but
|
|
133
|
+
// carry() (up to maxChars/2) plus a full-size piece could still land at
|
|
134
|
+
// ~1.5x, and an over-budget chunk blows the embedder's context for no
|
|
135
|
+
// benefit. Split rather than silently shipping an oversized chunk.
|
|
136
|
+
if (trimmed.length > maxChars) {
|
|
137
|
+
for (const part of splitLongText(trimmed, maxChars)) {
|
|
138
|
+
const t = part.trim();
|
|
139
|
+
if (!t)
|
|
140
|
+
continue;
|
|
141
|
+
chunks.push({ text: t, index: cursor++, tokenCount: estimateTokens(t) });
|
|
142
|
+
}
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
68
145
|
chunks.push({
|
|
69
146
|
text: trimmed,
|
|
70
147
|
index: cursor++,
|
|
@@ -72,9 +149,12 @@ export function chunkText(text, options = {}) {
|
|
|
72
149
|
});
|
|
73
150
|
};
|
|
74
151
|
const paragraphs = splitIntoParagraphs(text);
|
|
152
|
+
const overlap = Math.min(overlapChars, Math.floor(maxChars / 2));
|
|
153
|
+
/** The tail carried into the next chunk; "" when overlapChars is 0. */
|
|
154
|
+
const carry = (emitted) => overlap > 0 ? snapToWordStart(emitted.slice(-overlap), emitted.slice(-overlap - 1, -overlap)) : "";
|
|
75
155
|
let window = "";
|
|
76
156
|
for (const para of paragraphs) {
|
|
77
|
-
let pieces
|
|
157
|
+
let pieces;
|
|
78
158
|
if (para.length > maxChars) {
|
|
79
159
|
pieces = splitLongText(para, maxChars);
|
|
80
160
|
}
|
|
@@ -84,16 +164,33 @@ export function chunkText(text, options = {}) {
|
|
|
84
164
|
}
|
|
85
165
|
for (const piece of pieces) {
|
|
86
166
|
if (piece.length > maxChars) {
|
|
87
|
-
//
|
|
88
|
-
|
|
89
|
-
|
|
167
|
+
// A single sentence longer than maxChars: hard-split it. The pending
|
|
168
|
+
// window must be flushed first, otherwise the text collected so far is
|
|
169
|
+
// emitted twice (once here, once with the next piece appended).
|
|
170
|
+
const hards = splitLongText(piece, maxChars);
|
|
171
|
+
if (window) {
|
|
172
|
+
emit(window);
|
|
173
|
+
window = carry(window);
|
|
174
|
+
}
|
|
175
|
+
for (const hard of hards) {
|
|
176
|
+
const projectedHard = window.length + joiner(window, hard).length + hard.length;
|
|
177
|
+
if (window && projectedHard > maxChars) {
|
|
178
|
+
emit(window);
|
|
179
|
+
window = carry(window);
|
|
180
|
+
}
|
|
181
|
+
window += joiner(window, hard) + hard;
|
|
182
|
+
}
|
|
90
183
|
continue;
|
|
91
184
|
}
|
|
92
|
-
|
|
185
|
+
const projected = window.length + joiner(window, piece).length + piece.length;
|
|
186
|
+
if (window && projected > maxChars) {
|
|
93
187
|
emit(window);
|
|
94
|
-
|
|
188
|
+
// Overlap carries the tail of the previous chunk. overlap === 0 must
|
|
189
|
+
// reset the window: slice(-0) returns the whole string, which used to
|
|
190
|
+
// duplicate the previous chunk and grow it without bound.
|
|
191
|
+
window = carry(window);
|
|
95
192
|
}
|
|
96
|
-
window += (window
|
|
193
|
+
window += joiner(window, piece) + piece;
|
|
97
194
|
}
|
|
98
195
|
}
|
|
99
196
|
if (window.trim())
|
|
@@ -106,4 +203,15 @@ export function estimateTokens(text) {
|
|
|
106
203
|
const rest = text.length - cjk;
|
|
107
204
|
return Math.ceil(cjk * 1.5) + Math.ceil(rest / 4);
|
|
108
205
|
}
|
|
206
|
+
/**
|
|
207
|
+
* Same estimate for an already-known character count. Lets a caller total
|
|
208
|
+
* LENGTH() across many rows in SQL instead of materialising each content
|
|
209
|
+
* blob just to re-count its characters. Assumes no CJK in the total, which
|
|
210
|
+
* under-counts CJK-heavy stores — the same tradeoff the SQL path already made.
|
|
211
|
+
*/
|
|
212
|
+
export function estimateTokensFromChars(chars) {
|
|
213
|
+
if (!Number.isFinite(chars) || chars <= 0)
|
|
214
|
+
return 0;
|
|
215
|
+
return Math.ceil(chars / 4);
|
|
216
|
+
}
|
|
109
217
|
//# sourceMappingURL=chunker.js.map
|
package/dist/rag/chunker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,YAAY,GAAG,
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,6DAA6D,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACvF,IAAI,wCAAwC,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,+FAA+F,CAAC,IAAI,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,wEAAwE;QACxE,4EAA4E;QAC5E,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY;YAAE,MAAM,GAAG,CAAC,MAAM,CAAC;QAEnC,IAAI,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;YACvB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/B,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,SAAS;QACX,CAAC;QAED,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,KAAa;IACvC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,WAAW,GAAG,QAAQ,CAAC;IAC7B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,SAAS;QACxC,sEAAsE;QACtE,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,SAAS;QAC3F,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAAE,GAAG,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,KAAK,GAAG,GAAG,CAAC;QACZ,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,aAAqB;IAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,IAAI,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;AAC7D,CAAC;AAED,mFAAmF;AACnF,SAAS,SAAS,CAAC,SAAiB,EAAE,GAAW;IAC/C,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACjF,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAC5B,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,EAC1B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,EAC3B,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAC5B,CAAC;QACF,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACtC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,SAAS,MAAM,CAAC,IAAY,EAAE,KAAa;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,WAAW,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC;IAC7D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IACzG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,sCAAsC,CAAC,CAAC;IACrH,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,MAAM,IAAI,GAAG,CAAC,OAAe,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,uEAAuE;QACvE,wEAAwE;QACxE,sEAAsE;QACtE,mEAAmE;QACnE,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACpD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;gBACtB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,EAAE;YACf,UAAU,EAAE,cAAc,CAAC,OAAO,CAAC;SACpC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;IACjE,uEAAuE;IACvE,MAAM,KAAK,GAAG,CAAC,OAAe,EAAE,EAAE,CAChC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAErG,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3B,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC5B,qEAAqE;gBACrE,uEAAuE;gBACvE,gEAAgE;gBAChE,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,CAAC;oBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBACzB,CAAC;gBACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;oBAChF,IAAI,MAAM,IAAI,aAAa,GAAG,QAAQ,EAAE,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACb,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBACzB,CAAC;oBACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC;gBACxC,CAAC;gBACD,SAAS;YACX,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YAC9E,IAAI,MAAM,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;gBACnC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACb,qEAAqE;gBACrE,sEAAsE;gBACtE,0DAA0D;gBAC1D,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa;IACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,aAAa,IAAI;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM3F;
|
|
1
|
+
{"version":3,"file":"embeddings.d.ts","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAaA,eAAO,MAAM,eAAe,OAAO,CAAC;AAEpC,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,cAAc,CAAC;AAEzD,eAAO,MAAM,kBAAkB,EAAE,iBAC6C,CAAC;AAE/E,eAAO,MAAM,eAAe,QAA2D,CAAC;AAYxF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAED,wBAAgB,aAAa,IAAI;IAAE,QAAQ,EAAE,iBAAiB,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM3F;AAyBD,4EAA4E;AAC5E,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CA0BnE"}
|
package/dist/rag/embeddings.js
CHANGED
|
@@ -39,6 +39,8 @@ export function embeddingInfo() {
|
|
|
39
39
|
let extractorPromise = null;
|
|
40
40
|
async function getExtractor() {
|
|
41
41
|
if (!extractorPromise) {
|
|
42
|
+
// A failed model load must not be cached for the process lifetime: reset on
|
|
43
|
+
// rejection so a transient network/first-download error can be retried.
|
|
42
44
|
extractorPromise = (async () => {
|
|
43
45
|
// @ts-expect-error — @huggingface/transformers is an optional dependency
|
|
44
46
|
const mod = await import("@huggingface/transformers");
|
|
@@ -48,6 +50,9 @@ async function getExtractor() {
|
|
|
48
50
|
}
|
|
49
51
|
return pipeline("feature-extraction", EMBEDDING_MODEL, { quantized: true });
|
|
50
52
|
})();
|
|
53
|
+
extractorPromise.catch(() => {
|
|
54
|
+
extractorPromise = null;
|
|
55
|
+
});
|
|
51
56
|
}
|
|
52
57
|
return extractorPromise;
|
|
53
58
|
}
|
|
@@ -65,6 +70,15 @@ export async function embedText(text) {
|
|
|
65
70
|
throw new Error("transformers embed returned an unexpected shape");
|
|
66
71
|
}
|
|
67
72
|
const vec = Float64Array.from(data);
|
|
73
|
+
// A NaN/Inf from the model poisons every score it touches: dot products become
|
|
74
|
+
// NaN, comparisons go false, and the row silently stops matching anything —
|
|
75
|
+
// an invisible, permanent retrieval hole. Reject it at the boundary instead.
|
|
76
|
+
for (let i = 0; i < vec.length; i++) {
|
|
77
|
+
if (!Number.isFinite(vec[i])) {
|
|
78
|
+
throw new Error(`Embedding for this input contains a non-finite value at index ${i}. ` +
|
|
79
|
+
`The input is likely empty or degenerate; retry with more text or set EMBEDDING_PROVIDER=local.`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
68
82
|
ensureDim(vec, modelName());
|
|
69
83
|
return vec;
|
|
70
84
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/rag/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE3D,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AAIpC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC;AAE/E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,yBAAyB,CAAC;AAExF,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,yBAAyB,EAAE,GAAG;IAC9B,8CAA8C,EAAE,GAAG;IACnD,8BAA8B,EAAE,GAAG;IACnC,kBAAkB,EAAE,GAAG;IACvB,0BAA0B,EAAE,GAAG;IAC/B,6BAA6B,EAAE,GAAG;CACnC,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,OAAO,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,eAAe,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO;QACL,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,IAAI,eAAe;QACtD,KAAK,EAAE,SAAS,EAAE;KACnB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,4EAA4E;QAC5E,wEAAwE;QACxE,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,yEAAyE;YACzE,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;YACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAmG,CAAC;YACzH,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,QAAQ,CAAC,oBAAoB,EAAE,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9E,CAAC,CAAC,EAAE,CAAC;QACL,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1B,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,IAAI,kBAAkB,KAAK,OAAO,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5B,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1E,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,IAAyB,CAAC,CAAC;IACzD,+EAA+E;IAC/E,4EAA4E;IAC5E,6EAA6E;IAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,iEAAiE,CAAC,IAAI;gBACpE,gGAAgG,CACnG,CAAC;QACJ,CAAC;IACH,CAAC;IACD,SAAS,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IAC5B,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/rag/pipeline.d.ts
CHANGED
|
@@ -26,21 +26,38 @@ export interface IngestFileOptions {
|
|
|
26
26
|
source?: string;
|
|
27
27
|
contentType?: string;
|
|
28
28
|
metadata?: Record<string, unknown>;
|
|
29
|
+
/** Namespace the dedup hash, so equal text from different scopes is kept apart. */
|
|
30
|
+
dedupScope?: string;
|
|
29
31
|
}
|
|
30
32
|
export interface IngestDirectoryOptions {
|
|
31
33
|
extensions?: string[];
|
|
32
34
|
recursive?: boolean;
|
|
33
35
|
metadata?: Record<string, unknown>;
|
|
34
36
|
}
|
|
37
|
+
/** Max content accepted from a tool call or CLI, so one payload cannot stall the server for a minute. */
|
|
38
|
+
export declare const MAX_INGEST_CHARS = 1000000;
|
|
35
39
|
export declare function ingestText(content: string, title: string, opts?: IngestFileOptions): Promise<IngestResult>;
|
|
36
40
|
export declare function ingestFile(filePath: string, opts?: IngestFileOptions): Promise<IngestResult>;
|
|
37
41
|
export declare function ingestDirectory(dirPath: string, opts?: IngestDirectoryOptions): Promise<{
|
|
38
42
|
ingested: IngestResult[];
|
|
39
43
|
skipped: string[];
|
|
44
|
+
ingestedCount: number;
|
|
45
|
+
skippedCount: number;
|
|
40
46
|
}>;
|
|
41
47
|
export declare function searchDocs(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<SearchHit[]>;
|
|
42
48
|
export declare function retrieve(query: string, topK?: number, minScore?: number, filters?: SearchChunkFilters): Promise<RetrievedContext>;
|
|
43
|
-
|
|
49
|
+
/** Bounded by default: a big store would otherwise dump every row into one tool result. */
|
|
50
|
+
export declare const LIST_DOCS_DEFAULT = 100;
|
|
51
|
+
export declare const LIST_DOCS_MAX = 1000;
|
|
52
|
+
export declare function listDocuments(opts?: {
|
|
53
|
+
limit?: number;
|
|
54
|
+
offset?: number;
|
|
55
|
+
}): {
|
|
56
|
+
documents: Array<Record<string, unknown>>;
|
|
57
|
+
total: number;
|
|
58
|
+
limit: number;
|
|
59
|
+
offset: number;
|
|
60
|
+
};
|
|
44
61
|
export declare function deleteDocument(docId: string): {
|
|
45
62
|
deleted: boolean;
|
|
46
63
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AACvF,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5G,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAgCD,yGAAyG;AACzG,eAAO,MAAM,gBAAgB,UAAY,CAAC;AAE1C,wBAAsB,UAAU,CAC9B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,YAAY,CAAC,CAgHvB;AA6CD,wBAAsB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,GAAE,iBAAsB,GAAG,OAAO,CAAC,YAAY,CAAC,CAuBtG;AA2BD,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC;IAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAAC,CA+DvG;AAED,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,SAAK,EAAE,QAAQ,SAAO,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAG9H;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,IAAI,SAAI,EACR,QAAQ,SAAO,EACf,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAa3B;AAED,2FAA2F;AAC3F,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,aAAa,OAAO,CAAC;AAElC,wBAAgB,aAAa,CAAC,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG;IAC7E,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAeA;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,CAkBlE;AAED,wBAAgB,aAAa,IAAI;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAMrF;AAYD,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAC"}
|