@dudousxd/nestjs-agent-rag 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +78 -0
- package/dist/index.cjs +883 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +300 -0
- package/dist/index.d.ts +300 -0
- package/dist/index.js +845 -0
- package/dist/index.js.map +1 -0
- package/package.json +48 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,845 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
|
|
3
|
+
|
|
4
|
+
// src/chunk.ts
|
|
5
|
+
function chunkText(text, options = {}) {
|
|
6
|
+
const chunkSize = options.chunkSize ?? 800;
|
|
7
|
+
const overlap = options.overlap ?? 100;
|
|
8
|
+
const normalized = text.trim();
|
|
9
|
+
if (normalized.length === 0) {
|
|
10
|
+
return [];
|
|
11
|
+
}
|
|
12
|
+
if (normalized.length <= chunkSize) {
|
|
13
|
+
return [
|
|
14
|
+
normalized
|
|
15
|
+
];
|
|
16
|
+
}
|
|
17
|
+
const chunks = [];
|
|
18
|
+
let start = 0;
|
|
19
|
+
while (start < normalized.length) {
|
|
20
|
+
const end = Math.min(start + chunkSize, normalized.length);
|
|
21
|
+
let breakAt = end;
|
|
22
|
+
if (end < normalized.length) {
|
|
23
|
+
const window = normalized.slice(start, end);
|
|
24
|
+
const boundary = Math.max(window.lastIndexOf("\n\n"), window.lastIndexOf(". "), window.lastIndexOf(" "));
|
|
25
|
+
if (boundary > chunkSize / 2) {
|
|
26
|
+
breakAt = start + boundary + 1;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
const chunk = normalized.slice(start, breakAt).trim();
|
|
30
|
+
if (chunk.length > 0) {
|
|
31
|
+
chunks.push(chunk);
|
|
32
|
+
}
|
|
33
|
+
if (breakAt >= normalized.length) {
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
start = Math.max(breakAt - overlap, start + 1);
|
|
37
|
+
}
|
|
38
|
+
return chunks;
|
|
39
|
+
}
|
|
40
|
+
__name(chunkText, "chunkText");
|
|
41
|
+
|
|
42
|
+
// src/vector-store.ts
|
|
43
|
+
function documentIdOf(chunkId) {
|
|
44
|
+
return chunkId.replace(/#\d+$/, "");
|
|
45
|
+
}
|
|
46
|
+
__name(documentIdOf, "documentIdOf");
|
|
47
|
+
|
|
48
|
+
// src/filter.ts
|
|
49
|
+
function matchesFilter(metadata, filter) {
|
|
50
|
+
if (metadata === void 0) {
|
|
51
|
+
return Object.keys(filter).length === 0;
|
|
52
|
+
}
|
|
53
|
+
return Object.entries(filter).every(([key, value]) => metadata[key] === value);
|
|
54
|
+
}
|
|
55
|
+
__name(matchesFilter, "matchesFilter");
|
|
56
|
+
|
|
57
|
+
// src/memory-vector-store.ts
|
|
58
|
+
var MemoryVectorStore = class {
|
|
59
|
+
static {
|
|
60
|
+
__name(this, "MemoryVectorStore");
|
|
61
|
+
}
|
|
62
|
+
records = /* @__PURE__ */ new Map();
|
|
63
|
+
async upsert(records) {
|
|
64
|
+
for (const record of records) {
|
|
65
|
+
this.records.set(record.id, record);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
async remove(documentId) {
|
|
69
|
+
for (const id of this.records.keys()) {
|
|
70
|
+
if (documentIdOf(id) === documentId) {
|
|
71
|
+
this.records.delete(id);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
async listDocuments(filter) {
|
|
76
|
+
const documents = /* @__PURE__ */ new Map();
|
|
77
|
+
for (const record of this.records.values()) {
|
|
78
|
+
if (filter !== void 0 && !matchesFilter(record.metadata, filter)) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const id = documentIdOf(record.id);
|
|
82
|
+
if (!documents.has(id)) {
|
|
83
|
+
documents.set(id, {
|
|
84
|
+
id,
|
|
85
|
+
...record.metadata !== void 0 ? {
|
|
86
|
+
metadata: record.metadata
|
|
87
|
+
} : {}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return [
|
|
92
|
+
...documents.values()
|
|
93
|
+
];
|
|
94
|
+
}
|
|
95
|
+
async search(embedding, options) {
|
|
96
|
+
const scored = [];
|
|
97
|
+
for (const record of this.records.values()) {
|
|
98
|
+
if (options.filter !== void 0 && !matchesFilter(record.metadata, options.filter)) {
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
scored.push({
|
|
102
|
+
id: record.id,
|
|
103
|
+
text: record.text,
|
|
104
|
+
score: cosineSimilarity(embedding, record.embedding),
|
|
105
|
+
...record.source !== void 0 ? {
|
|
106
|
+
source: record.source
|
|
107
|
+
} : {},
|
|
108
|
+
...record.metadata !== void 0 ? {
|
|
109
|
+
metadata: record.metadata
|
|
110
|
+
} : {}
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, options.topK);
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
function cosineSimilarity(a, b) {
|
|
117
|
+
let dot = 0;
|
|
118
|
+
let normA = 0;
|
|
119
|
+
let normB = 0;
|
|
120
|
+
const length = Math.min(a.length, b.length);
|
|
121
|
+
for (let index = 0; index < length; index += 1) {
|
|
122
|
+
const valueA = a[index] ?? 0;
|
|
123
|
+
const valueB = b[index] ?? 0;
|
|
124
|
+
dot += valueA * valueB;
|
|
125
|
+
normA += valueA * valueA;
|
|
126
|
+
normB += valueB * valueB;
|
|
127
|
+
}
|
|
128
|
+
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
|
129
|
+
return denominator === 0 ? 0 : dot / denominator;
|
|
130
|
+
}
|
|
131
|
+
__name(cosineSimilarity, "cosineSimilarity");
|
|
132
|
+
|
|
133
|
+
// src/pg-vector-store.ts
|
|
134
|
+
var PgVectorStore = class {
|
|
135
|
+
static {
|
|
136
|
+
__name(this, "PgVectorStore");
|
|
137
|
+
}
|
|
138
|
+
client;
|
|
139
|
+
table;
|
|
140
|
+
dimensions;
|
|
141
|
+
constructor(client, options = {}) {
|
|
142
|
+
this.client = client;
|
|
143
|
+
this.table = options.table ?? "agent_rag_chunks";
|
|
144
|
+
this.dimensions = options.dimensions ?? 1536;
|
|
145
|
+
}
|
|
146
|
+
/** Idempotent DDL: the `vector` extension, the chunks table, and the cosine HNSW index. */
|
|
147
|
+
async ensureSchema() {
|
|
148
|
+
await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
149
|
+
await this.client.query(`CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
150
|
+
id TEXT PRIMARY KEY,
|
|
151
|
+
text TEXT NOT NULL,
|
|
152
|
+
source TEXT,
|
|
153
|
+
metadata JSONB,
|
|
154
|
+
embedding vector(${this.dimensions}) NOT NULL
|
|
155
|
+
)`);
|
|
156
|
+
await this.client.query(`CREATE INDEX IF NOT EXISTS ${this.table}_embedding_idx
|
|
157
|
+
ON ${this.table} USING hnsw (embedding vector_cosine_ops)`);
|
|
158
|
+
}
|
|
159
|
+
async upsert(records) {
|
|
160
|
+
for (const record of records) {
|
|
161
|
+
await this.client.query(`INSERT INTO ${this.table} (id, text, source, metadata, embedding)
|
|
162
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
163
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
164
|
+
text = EXCLUDED.text,
|
|
165
|
+
source = EXCLUDED.source,
|
|
166
|
+
metadata = EXCLUDED.metadata,
|
|
167
|
+
embedding = EXCLUDED.embedding`, [
|
|
168
|
+
record.id,
|
|
169
|
+
record.text,
|
|
170
|
+
record.source ?? null,
|
|
171
|
+
record.metadata !== void 0 ? JSON.stringify(record.metadata) : null,
|
|
172
|
+
toVectorLiteral(record.embedding)
|
|
173
|
+
]);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async remove(documentId) {
|
|
177
|
+
await this.client.query(`DELETE FROM ${this.table} WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1`, [
|
|
178
|
+
documentId
|
|
179
|
+
]);
|
|
180
|
+
}
|
|
181
|
+
async listDocuments(filter) {
|
|
182
|
+
const hasFilter = filter !== void 0 && Object.keys(filter).length > 0;
|
|
183
|
+
const rows = await this.client.query(`SELECT DISTINCT ON (${DOCUMENT_ID_FROM_CHUNK}) ${DOCUMENT_ID_FROM_CHUNK} AS doc_id, metadata
|
|
184
|
+
FROM ${this.table}
|
|
185
|
+
${hasFilter ? "WHERE metadata @> $1::jsonb" : ""}
|
|
186
|
+
ORDER BY ${DOCUMENT_ID_FROM_CHUNK}`, hasFilter ? [
|
|
187
|
+
JSON.stringify(filter)
|
|
188
|
+
] : []);
|
|
189
|
+
return rows.map((row) => ({
|
|
190
|
+
id: row.doc_id,
|
|
191
|
+
...row.metadata !== null ? {
|
|
192
|
+
metadata: row.metadata
|
|
193
|
+
} : {}
|
|
194
|
+
}));
|
|
195
|
+
}
|
|
196
|
+
async search(embedding, options) {
|
|
197
|
+
const vector = toVectorLiteral(embedding);
|
|
198
|
+
const hasFilter = options.filter !== void 0 && Object.keys(options.filter).length > 0;
|
|
199
|
+
const rows = await this.client.query(`SELECT id, text, source, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
200
|
+
FROM ${this.table}
|
|
201
|
+
${hasFilter ? "WHERE metadata @> $3::jsonb" : ""}
|
|
202
|
+
ORDER BY embedding <=> $1::vector
|
|
203
|
+
LIMIT $2`, hasFilter ? [
|
|
204
|
+
vector,
|
|
205
|
+
options.topK,
|
|
206
|
+
JSON.stringify(options.filter)
|
|
207
|
+
] : [
|
|
208
|
+
vector,
|
|
209
|
+
options.topK
|
|
210
|
+
]);
|
|
211
|
+
return rows.map((row) => ({
|
|
212
|
+
id: row.id,
|
|
213
|
+
text: row.text,
|
|
214
|
+
score: Number(row.score),
|
|
215
|
+
...row.source !== null ? {
|
|
216
|
+
source: row.source
|
|
217
|
+
} : {},
|
|
218
|
+
...row.metadata !== null ? {
|
|
219
|
+
metadata: row.metadata
|
|
220
|
+
} : {}
|
|
221
|
+
}));
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
var DOCUMENT_ID_FROM_CHUNK = "regexp_replace(id, '#[0-9]+$', '')";
|
|
225
|
+
function toVectorLiteral(embedding) {
|
|
226
|
+
return `[${embedding.join(",")}]`;
|
|
227
|
+
}
|
|
228
|
+
__name(toVectorLiteral, "toVectorLiteral");
|
|
229
|
+
|
|
230
|
+
// src/redis-vector-store.ts
|
|
231
|
+
var RedisVectorStore = class {
|
|
232
|
+
static {
|
|
233
|
+
__name(this, "RedisVectorStore");
|
|
234
|
+
}
|
|
235
|
+
client;
|
|
236
|
+
index;
|
|
237
|
+
prefix;
|
|
238
|
+
dimensions;
|
|
239
|
+
filterableFields;
|
|
240
|
+
constructor(client, options = {}) {
|
|
241
|
+
this.client = client;
|
|
242
|
+
this.index = options.index ?? "agent_rag_idx";
|
|
243
|
+
this.prefix = options.prefix ?? "agent_rag:";
|
|
244
|
+
this.dimensions = options.dimensions ?? 1536;
|
|
245
|
+
this.filterableFields = options.filterableFields ?? [];
|
|
246
|
+
}
|
|
247
|
+
/** Idempotent: create the HNSW/cosine index if `FT.INFO` says it doesn't exist yet. */
|
|
248
|
+
async ensureSchema() {
|
|
249
|
+
try {
|
|
250
|
+
await this.client.sendCommand([
|
|
251
|
+
"FT.INFO",
|
|
252
|
+
this.index
|
|
253
|
+
]);
|
|
254
|
+
return;
|
|
255
|
+
} catch {
|
|
256
|
+
}
|
|
257
|
+
const schema = [
|
|
258
|
+
"text",
|
|
259
|
+
"TEXT",
|
|
260
|
+
"source",
|
|
261
|
+
"TAG",
|
|
262
|
+
"metadata_json",
|
|
263
|
+
"TEXT",
|
|
264
|
+
"NOINDEX"
|
|
265
|
+
];
|
|
266
|
+
for (const field of this.filterableFields) {
|
|
267
|
+
schema.push(`meta_${field}`, "TAG");
|
|
268
|
+
}
|
|
269
|
+
schema.push("embedding", "VECTOR", "HNSW", "6", "TYPE", "FLOAT32", "DIM", String(this.dimensions), "DISTANCE_METRIC", "COSINE");
|
|
270
|
+
await this.client.sendCommand([
|
|
271
|
+
"FT.CREATE",
|
|
272
|
+
this.index,
|
|
273
|
+
"ON",
|
|
274
|
+
"HASH",
|
|
275
|
+
"PREFIX",
|
|
276
|
+
"1",
|
|
277
|
+
this.prefix,
|
|
278
|
+
"SCHEMA",
|
|
279
|
+
...schema
|
|
280
|
+
]);
|
|
281
|
+
}
|
|
282
|
+
async upsert(records) {
|
|
283
|
+
for (const record of records) {
|
|
284
|
+
const args = [
|
|
285
|
+
"HSET",
|
|
286
|
+
`${this.prefix}${record.id}`,
|
|
287
|
+
"text",
|
|
288
|
+
record.text,
|
|
289
|
+
"embedding",
|
|
290
|
+
encodeVector(record.embedding)
|
|
291
|
+
];
|
|
292
|
+
if (record.source !== void 0) {
|
|
293
|
+
args.push("source", record.source);
|
|
294
|
+
}
|
|
295
|
+
if (record.metadata !== void 0) {
|
|
296
|
+
args.push("metadata_json", JSON.stringify(record.metadata));
|
|
297
|
+
for (const field of this.filterableFields) {
|
|
298
|
+
const value = record.metadata[field];
|
|
299
|
+
if (value !== void 0) {
|
|
300
|
+
args.push(`meta_${field}`, String(value));
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
await this.client.sendCommand(args);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
async remove(documentId) {
|
|
308
|
+
const keys = /* @__PURE__ */ new Set([
|
|
309
|
+
`${this.prefix}${documentId}`
|
|
310
|
+
]);
|
|
311
|
+
const pattern = `${this.prefix}${escapeGlob(documentId)}#*`;
|
|
312
|
+
let cursor = "0";
|
|
313
|
+
do {
|
|
314
|
+
const reply = await this.client.sendCommand([
|
|
315
|
+
"SCAN",
|
|
316
|
+
cursor,
|
|
317
|
+
"MATCH",
|
|
318
|
+
pattern,
|
|
319
|
+
"COUNT",
|
|
320
|
+
"256"
|
|
321
|
+
]);
|
|
322
|
+
const page = parseScanReply(reply);
|
|
323
|
+
cursor = page.cursor;
|
|
324
|
+
for (const key of page.keys) {
|
|
325
|
+
keys.add(key);
|
|
326
|
+
}
|
|
327
|
+
} while (cursor !== "0");
|
|
328
|
+
if (keys.size > 0) {
|
|
329
|
+
await this.client.sendCommand([
|
|
330
|
+
"DEL",
|
|
331
|
+
...keys
|
|
332
|
+
]);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
async listDocuments(filter) {
|
|
336
|
+
const query = buildFilter(filter);
|
|
337
|
+
const documents = /* @__PURE__ */ new Map();
|
|
338
|
+
const batchSize = 1e3;
|
|
339
|
+
let offset = 0;
|
|
340
|
+
for (; ; ) {
|
|
341
|
+
const reply = await this.client.sendCommand([
|
|
342
|
+
"FT.SEARCH",
|
|
343
|
+
this.index,
|
|
344
|
+
query,
|
|
345
|
+
"RETURN",
|
|
346
|
+
"1",
|
|
347
|
+
"metadata_json",
|
|
348
|
+
"LIMIT",
|
|
349
|
+
String(offset),
|
|
350
|
+
String(batchSize),
|
|
351
|
+
"DIALECT",
|
|
352
|
+
"2"
|
|
353
|
+
]);
|
|
354
|
+
const passages = parseSearchReply(reply, this.prefix);
|
|
355
|
+
for (const passage of passages) {
|
|
356
|
+
const id = documentIdOf(passage.id);
|
|
357
|
+
if (!documents.has(id)) {
|
|
358
|
+
documents.set(id, {
|
|
359
|
+
id,
|
|
360
|
+
...passage.metadata !== void 0 ? {
|
|
361
|
+
metadata: passage.metadata
|
|
362
|
+
} : {}
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
if (passages.length < batchSize) {
|
|
367
|
+
break;
|
|
368
|
+
}
|
|
369
|
+
offset += batchSize;
|
|
370
|
+
}
|
|
371
|
+
return [
|
|
372
|
+
...documents.values()
|
|
373
|
+
];
|
|
374
|
+
}
|
|
375
|
+
async search(embedding, options) {
|
|
376
|
+
const query = `${buildFilter(options.filter)}=>[KNN ${options.topK} @embedding $BLOB AS vector_score]`;
|
|
377
|
+
const reply = await this.client.sendCommand([
|
|
378
|
+
"FT.SEARCH",
|
|
379
|
+
this.index,
|
|
380
|
+
query,
|
|
381
|
+
"PARAMS",
|
|
382
|
+
"2",
|
|
383
|
+
"BLOB",
|
|
384
|
+
encodeVector(embedding),
|
|
385
|
+
"RETURN",
|
|
386
|
+
"4",
|
|
387
|
+
"text",
|
|
388
|
+
"source",
|
|
389
|
+
"metadata_json",
|
|
390
|
+
"vector_score",
|
|
391
|
+
"SORTBY",
|
|
392
|
+
"vector_score",
|
|
393
|
+
"ASC",
|
|
394
|
+
"DIALECT",
|
|
395
|
+
"2",
|
|
396
|
+
"LIMIT",
|
|
397
|
+
"0",
|
|
398
|
+
String(options.topK)
|
|
399
|
+
]);
|
|
400
|
+
return parseSearchReply(reply, this.prefix);
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
function encodeVector(embedding) {
|
|
404
|
+
return Buffer.from(new Float32Array(embedding).buffer);
|
|
405
|
+
}
|
|
406
|
+
__name(encodeVector, "encodeVector");
|
|
407
|
+
function buildFilter(filter) {
|
|
408
|
+
if (filter === void 0 || Object.keys(filter).length === 0) {
|
|
409
|
+
return "*";
|
|
410
|
+
}
|
|
411
|
+
const clauses = Object.entries(filter).map(([key, value]) => `@meta_${key}:{${escapeTag(String(value))}}`);
|
|
412
|
+
return `(${clauses.join(" ")})`;
|
|
413
|
+
}
|
|
414
|
+
__name(buildFilter, "buildFilter");
|
|
415
|
+
function escapeTag(value) {
|
|
416
|
+
return value.replace(/[^a-zA-Z0-9_]/g, (char) => `\\${char}`);
|
|
417
|
+
}
|
|
418
|
+
__name(escapeTag, "escapeTag");
|
|
419
|
+
function escapeGlob(value) {
|
|
420
|
+
return value.replace(/[*?[\]\\]/g, (char) => `\\${char}`);
|
|
421
|
+
}
|
|
422
|
+
__name(escapeGlob, "escapeGlob");
|
|
423
|
+
function parseScanReply(reply) {
|
|
424
|
+
if (Array.isArray(reply) && reply.length >= 2) {
|
|
425
|
+
const rawKeys = reply[1];
|
|
426
|
+
return {
|
|
427
|
+
cursor: toStr(reply[0]),
|
|
428
|
+
keys: Array.isArray(rawKeys) ? rawKeys.map(toStr) : []
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
if (typeof reply === "object" && reply !== null && "cursor" in reply && "keys" in reply) {
|
|
432
|
+
const { cursor, keys } = reply;
|
|
433
|
+
return {
|
|
434
|
+
cursor: toStr(cursor),
|
|
435
|
+
keys: Array.isArray(keys) ? keys.map(toStr) : []
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
return {
|
|
439
|
+
cursor: "0",
|
|
440
|
+
keys: []
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
__name(parseScanReply, "parseScanReply");
|
|
444
|
+
function toStr(value) {
|
|
445
|
+
if (typeof value === "string") {
|
|
446
|
+
return value;
|
|
447
|
+
}
|
|
448
|
+
return Buffer.isBuffer(value) ? value.toString("utf8") : String(value);
|
|
449
|
+
}
|
|
450
|
+
__name(toStr, "toStr");
|
|
451
|
+
function parseSearchReply(reply, prefix) {
|
|
452
|
+
if (Array.isArray(reply)) {
|
|
453
|
+
const passages = [];
|
|
454
|
+
for (let index = 1; index + 1 < reply.length; index += 2) {
|
|
455
|
+
const rawFields = reply[index + 1];
|
|
456
|
+
if (Array.isArray(rawFields)) {
|
|
457
|
+
const attrs = {};
|
|
458
|
+
for (let field = 0; field + 1 < rawFields.length; field += 2) {
|
|
459
|
+
attrs[toStr(rawFields[field])] = toStr(rawFields[field + 1]);
|
|
460
|
+
}
|
|
461
|
+
passages.push(toPassage(toStr(reply[index]), attrs, prefix));
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return passages;
|
|
465
|
+
}
|
|
466
|
+
if (typeof reply === "object" && reply !== null && "results" in reply) {
|
|
467
|
+
const results = reply.results;
|
|
468
|
+
if (Array.isArray(results)) {
|
|
469
|
+
return results.map((result) => toPassage(readId(result), readAttributes(result), prefix));
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
return [];
|
|
473
|
+
}
|
|
474
|
+
__name(parseSearchReply, "parseSearchReply");
|
|
475
|
+
function readId(result) {
|
|
476
|
+
if (typeof result === "object" && result !== null && "id" in result) {
|
|
477
|
+
return toStr(result.id);
|
|
478
|
+
}
|
|
479
|
+
return "";
|
|
480
|
+
}
|
|
481
|
+
__name(readId, "readId");
|
|
482
|
+
function readAttributes(result) {
|
|
483
|
+
const attrs = {};
|
|
484
|
+
if (typeof result === "object" && result !== null && "extra_attributes" in result) {
|
|
485
|
+
const raw = result.extra_attributes;
|
|
486
|
+
if (typeof raw === "object" && raw !== null) {
|
|
487
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
488
|
+
attrs[key] = toStr(value);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return attrs;
|
|
493
|
+
}
|
|
494
|
+
__name(readAttributes, "readAttributes");
|
|
495
|
+
function toPassage(rawId, attrs, prefix) {
|
|
496
|
+
const distance = Number(attrs.vector_score ?? "1");
|
|
497
|
+
const source = attrs.source;
|
|
498
|
+
return {
|
|
499
|
+
id: rawId.startsWith(prefix) ? rawId.slice(prefix.length) : rawId,
|
|
500
|
+
text: attrs.text ?? "",
|
|
501
|
+
score: 1 - distance,
|
|
502
|
+
...source !== void 0 ? {
|
|
503
|
+
source
|
|
504
|
+
} : {},
|
|
505
|
+
...parseMetadata(attrs.metadata_json)
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
__name(toPassage, "toPassage");
|
|
509
|
+
function parseMetadata(raw) {
|
|
510
|
+
if (raw === void 0) {
|
|
511
|
+
return {};
|
|
512
|
+
}
|
|
513
|
+
try {
|
|
514
|
+
const parsed = JSON.parse(raw);
|
|
515
|
+
if (typeof parsed === "object" && parsed !== null) {
|
|
516
|
+
return {
|
|
517
|
+
metadata: parsed
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
} catch {
|
|
521
|
+
}
|
|
522
|
+
return {};
|
|
523
|
+
}
|
|
524
|
+
__name(parseMetadata, "parseMetadata");
|
|
525
|
+
|
|
526
|
+
// src/embedding-retriever.ts
|
|
527
|
+
var EmbeddingRetriever = class {
|
|
528
|
+
static {
|
|
529
|
+
__name(this, "EmbeddingRetriever");
|
|
530
|
+
}
|
|
531
|
+
embedder;
|
|
532
|
+
store;
|
|
533
|
+
constructor(embedder, store) {
|
|
534
|
+
this.embedder = embedder;
|
|
535
|
+
this.store = store;
|
|
536
|
+
}
|
|
537
|
+
async retrieve(query, options = {}) {
|
|
538
|
+
const [embedding] = await this.embedder.embed([
|
|
539
|
+
query
|
|
540
|
+
]);
|
|
541
|
+
if (embedding === void 0) {
|
|
542
|
+
return [];
|
|
543
|
+
}
|
|
544
|
+
return this.store.search(embedding, {
|
|
545
|
+
topK: options.topK ?? 5,
|
|
546
|
+
...options.filter !== void 0 ? {
|
|
547
|
+
filter: options.filter
|
|
548
|
+
} : {}
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
};
|
|
552
|
+
|
|
553
|
+
// src/keyword-retriever.ts
|
|
554
|
+
var KeywordRetriever = class {
|
|
555
|
+
static {
|
|
556
|
+
__name(this, "KeywordRetriever");
|
|
557
|
+
}
|
|
558
|
+
docs = /* @__PURE__ */ new Map();
|
|
559
|
+
documentFrequency = /* @__PURE__ */ new Map();
|
|
560
|
+
totalLength = 0;
|
|
561
|
+
k1;
|
|
562
|
+
b;
|
|
563
|
+
constructor(options = {}) {
|
|
564
|
+
this.k1 = options.k1 ?? 1.5;
|
|
565
|
+
this.b = options.b ?? 0.75;
|
|
566
|
+
}
|
|
567
|
+
/** Index (or re-index) records. Re-adding an existing id replaces its posting and updates stats. */
|
|
568
|
+
add(records) {
|
|
569
|
+
for (const record of records) {
|
|
570
|
+
this.remove(record.id);
|
|
571
|
+
const tokens = tokenize(record.text);
|
|
572
|
+
const frequencies = /* @__PURE__ */ new Map();
|
|
573
|
+
for (const token of tokens) {
|
|
574
|
+
frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
|
|
575
|
+
}
|
|
576
|
+
for (const term of frequencies.keys()) {
|
|
577
|
+
this.documentFrequency.set(term, (this.documentFrequency.get(term) ?? 0) + 1);
|
|
578
|
+
}
|
|
579
|
+
this.docs.set(record.id, {
|
|
580
|
+
record,
|
|
581
|
+
frequencies,
|
|
582
|
+
length: tokens.length
|
|
583
|
+
});
|
|
584
|
+
this.totalLength += tokens.length;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
remove(id) {
|
|
588
|
+
const existing = this.docs.get(id);
|
|
589
|
+
if (existing === void 0) {
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
for (const term of existing.frequencies.keys()) {
|
|
593
|
+
const next = (this.documentFrequency.get(term) ?? 1) - 1;
|
|
594
|
+
if (next <= 0) {
|
|
595
|
+
this.documentFrequency.delete(term);
|
|
596
|
+
} else {
|
|
597
|
+
this.documentFrequency.set(term, next);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
this.totalLength -= existing.length;
|
|
601
|
+
this.docs.delete(id);
|
|
602
|
+
}
|
|
603
|
+
async retrieve(query, options = {}) {
|
|
604
|
+
const total = this.docs.size;
|
|
605
|
+
if (total === 0) {
|
|
606
|
+
return [];
|
|
607
|
+
}
|
|
608
|
+
const averageLength = this.totalLength / total;
|
|
609
|
+
const queryTerms = new Set(tokenize(query));
|
|
610
|
+
const scored = [];
|
|
611
|
+
for (const { record, frequencies, length } of this.docs.values()) {
|
|
612
|
+
if (options.filter !== void 0 && !matchesFilter(record.metadata, options.filter)) {
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
let score = 0;
|
|
616
|
+
for (const term of queryTerms) {
|
|
617
|
+
const termFrequency = frequencies.get(term);
|
|
618
|
+
if (termFrequency === void 0) {
|
|
619
|
+
continue;
|
|
620
|
+
}
|
|
621
|
+
const df = this.documentFrequency.get(term) ?? 0;
|
|
622
|
+
const idf = Math.log(1 + (total - df + 0.5) / (df + 0.5));
|
|
623
|
+
const denominator = termFrequency + this.k1 * (1 - this.b + this.b * length / averageLength);
|
|
624
|
+
score += idf * (termFrequency * (this.k1 + 1) / denominator);
|
|
625
|
+
}
|
|
626
|
+
if (score > 0) {
|
|
627
|
+
scored.push({
|
|
628
|
+
id: record.id,
|
|
629
|
+
text: record.text,
|
|
630
|
+
score,
|
|
631
|
+
...record.source !== void 0 ? {
|
|
632
|
+
source: record.source
|
|
633
|
+
} : {},
|
|
634
|
+
...record.metadata !== void 0 ? {
|
|
635
|
+
metadata: record.metadata
|
|
636
|
+
} : {}
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, options.topK ?? 5);
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
function tokenize(text) {
|
|
644
|
+
return text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
|
|
645
|
+
}
|
|
646
|
+
__name(tokenize, "tokenize");
|
|
647
|
+
|
|
648
|
+
// src/hybrid-retriever.ts
|
|
649
|
+
var HybridRetriever = class {
|
|
650
|
+
static {
|
|
651
|
+
__name(this, "HybridRetriever");
|
|
652
|
+
}
|
|
653
|
+
retrievers;
|
|
654
|
+
options;
|
|
655
|
+
constructor(retrievers, options = {}) {
|
|
656
|
+
this.retrievers = retrievers;
|
|
657
|
+
this.options = options;
|
|
658
|
+
}
|
|
659
|
+
async retrieve(query, options = {}) {
|
|
660
|
+
const k = this.options.k ?? 60;
|
|
661
|
+
const fetchTopK = this.options.fetchTopK ?? 20;
|
|
662
|
+
const weights = this.options.weights;
|
|
663
|
+
const lists = await Promise.all(this.retrievers.map((retriever) => retriever.retrieve(query, {
|
|
664
|
+
topK: fetchTopK,
|
|
665
|
+
...options.filter !== void 0 ? {
|
|
666
|
+
filter: options.filter
|
|
667
|
+
} : {}
|
|
668
|
+
})));
|
|
669
|
+
const fused = /* @__PURE__ */ new Map();
|
|
670
|
+
lists.forEach((list, listIndex) => {
|
|
671
|
+
const weight = weights?.[listIndex] ?? 1;
|
|
672
|
+
list.forEach((passage, rank) => {
|
|
673
|
+
const contribution = weight / (k + rank + 1);
|
|
674
|
+
const existing = fused.get(passage.id);
|
|
675
|
+
if (existing !== void 0) {
|
|
676
|
+
existing.score += contribution;
|
|
677
|
+
} else {
|
|
678
|
+
fused.set(passage.id, {
|
|
679
|
+
passage,
|
|
680
|
+
score: contribution
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
});
|
|
685
|
+
return [
|
|
686
|
+
...fused.values()
|
|
687
|
+
].sort((a, b) => b.score - a.score).slice(0, options.topK ?? 5).map((entry) => ({
|
|
688
|
+
...entry.passage,
|
|
689
|
+
score: entry.score
|
|
690
|
+
}));
|
|
691
|
+
}
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
// src/reranking-retriever.ts
|
|
695
|
+
var RerankingRetriever = class {
|
|
696
|
+
static {
|
|
697
|
+
__name(this, "RerankingRetriever");
|
|
698
|
+
}
|
|
699
|
+
base;
|
|
700
|
+
reranker;
|
|
701
|
+
options;
|
|
702
|
+
constructor(base, reranker, options = {}) {
|
|
703
|
+
this.base = base;
|
|
704
|
+
this.reranker = reranker;
|
|
705
|
+
this.options = options;
|
|
706
|
+
}
|
|
707
|
+
async retrieve(query, options = {}) {
|
|
708
|
+
const candidates = await this.base.retrieve(query, {
|
|
709
|
+
topK: this.options.fetchTopK ?? 20,
|
|
710
|
+
...options.filter !== void 0 ? {
|
|
711
|
+
filter: options.filter
|
|
712
|
+
} : {}
|
|
713
|
+
});
|
|
714
|
+
return this.reranker.rerank(query, candidates, {
|
|
715
|
+
topK: options.topK ?? 5
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
};
|
|
719
|
+
|
|
720
|
+
// src/filtered-retriever.ts
|
|
721
|
+
var FilteredRetriever = class {
|
|
722
|
+
static {
|
|
723
|
+
__name(this, "FilteredRetriever");
|
|
724
|
+
}
|
|
725
|
+
base;
|
|
726
|
+
filter;
|
|
727
|
+
constructor(base, filter) {
|
|
728
|
+
this.base = base;
|
|
729
|
+
this.filter = filter;
|
|
730
|
+
}
|
|
731
|
+
async retrieve(query, options = {}) {
|
|
732
|
+
return this.base.retrieve(query, {
|
|
733
|
+
...options,
|
|
734
|
+
filter: {
|
|
735
|
+
...options.filter,
|
|
736
|
+
...this.filter
|
|
737
|
+
}
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
};
|
|
741
|
+
|
|
742
|
+
// src/ingest.ts
|
|
743
|
+
function chunkDocuments(documents, options = {}) {
|
|
744
|
+
const chunks = [];
|
|
745
|
+
for (const document of documents) {
|
|
746
|
+
chunkText(document.text, options).forEach((text, index) => {
|
|
747
|
+
chunks.push({
|
|
748
|
+
id: `${document.id}#${index}`,
|
|
749
|
+
text,
|
|
750
|
+
...document.source !== void 0 ? {
|
|
751
|
+
source: document.source
|
|
752
|
+
} : {},
|
|
753
|
+
...document.metadata !== void 0 ? {
|
|
754
|
+
metadata: document.metadata
|
|
755
|
+
} : {}
|
|
756
|
+
});
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
return chunks;
|
|
760
|
+
}
|
|
761
|
+
__name(chunkDocuments, "chunkDocuments");
|
|
762
|
+
async function ingestChunks(chunks, options) {
|
|
763
|
+
if (chunks.length === 0) {
|
|
764
|
+
return 0;
|
|
765
|
+
}
|
|
766
|
+
const embeddings = await options.embedder.embed(chunks.map((chunk) => chunk.text));
|
|
767
|
+
const records = chunks.map((chunk, index) => ({
|
|
768
|
+
...chunk,
|
|
769
|
+
embedding: embeddings[index] ?? []
|
|
770
|
+
}));
|
|
771
|
+
await options.store.upsert(records);
|
|
772
|
+
return records.length;
|
|
773
|
+
}
|
|
774
|
+
__name(ingestChunks, "ingestChunks");
|
|
775
|
+
async function ingestDocuments(documents, options) {
|
|
776
|
+
return ingestChunks(chunkDocuments(documents, options), options);
|
|
777
|
+
}
|
|
778
|
+
__name(ingestDocuments, "ingestDocuments");
|
|
779
|
+
|
|
780
|
+
// src/retrieval-tool.ts
|
|
781
|
+
function isQueryInput(value) {
|
|
782
|
+
return typeof value === "object" && value !== null && "query" in value && typeof value.query === "string";
|
|
783
|
+
}
|
|
784
|
+
__name(isQueryInput, "isQueryInput");
|
|
785
|
+
var querySchema = {
|
|
786
|
+
"~standard": {
|
|
787
|
+
version: 1,
|
|
788
|
+
vendor: "nestjs-agent-rag",
|
|
789
|
+
validate(value) {
|
|
790
|
+
return isQueryInput(value) ? {
|
|
791
|
+
value: {
|
|
792
|
+
query: value.query
|
|
793
|
+
}
|
|
794
|
+
} : {
|
|
795
|
+
issues: [
|
|
796
|
+
{
|
|
797
|
+
message: "retrieval tool expects { query: string }"
|
|
798
|
+
}
|
|
799
|
+
]
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
};
|
|
804
|
+
function createRetrievalTool(retriever, options = {}) {
|
|
805
|
+
const topK = options.topK;
|
|
806
|
+
return {
|
|
807
|
+
spec: {
|
|
808
|
+
name: options.name ?? "search_knowledge",
|
|
809
|
+
kind: "read",
|
|
810
|
+
description: options.description ?? "Search the knowledge base for passages relevant to a query. Returns passages with their source for citation.",
|
|
811
|
+
inputSchema: querySchema
|
|
812
|
+
},
|
|
813
|
+
handler: {
|
|
814
|
+
async execute(input) {
|
|
815
|
+
if (!isQueryInput(input)) {
|
|
816
|
+
throw new Error("retrieval tool expects { query: string }");
|
|
817
|
+
}
|
|
818
|
+
const passages = await retriever.retrieve(input.query, topK !== void 0 ? {
|
|
819
|
+
topK
|
|
820
|
+
} : {});
|
|
821
|
+
return {
|
|
822
|
+
passages
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
__name(createRetrievalTool, "createRetrievalTool");
|
|
829
|
+
export {
|
|
830
|
+
EmbeddingRetriever,
|
|
831
|
+
FilteredRetriever,
|
|
832
|
+
HybridRetriever,
|
|
833
|
+
KeywordRetriever,
|
|
834
|
+
MemoryVectorStore,
|
|
835
|
+
PgVectorStore,
|
|
836
|
+
RedisVectorStore,
|
|
837
|
+
RerankingRetriever,
|
|
838
|
+
chunkDocuments,
|
|
839
|
+
chunkText,
|
|
840
|
+
createRetrievalTool,
|
|
841
|
+
documentIdOf,
|
|
842
|
+
ingestChunks,
|
|
843
|
+
ingestDocuments
|
|
844
|
+
};
|
|
845
|
+
//# sourceMappingURL=index.js.map
|