@cogitator-ai/rag 0.1.10 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -31
- package/dist/chunkers/create-chunker.js.map +1 -1
- package/dist/chunkers/index.d.ts.map +1 -1
- package/dist/chunkers/index.js.map +1 -1
- package/dist/chunkers/recursive-chunker.d.ts +1 -2
- package/dist/chunkers/recursive-chunker.d.ts.map +1 -1
- package/dist/chunkers/recursive-chunker.js +68 -58
- package/dist/chunkers/recursive-chunker.js.map +1 -1
- package/dist/chunkers/semantic-chunker.d.ts.map +1 -1
- package/dist/chunkers/semantic-chunker.js +4 -1
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/loaders/csv-loader.d.ts.map +1 -1
- package/dist/loaders/csv-loader.js +9 -4
- package/dist/loaders/csv-loader.js.map +1 -1
- package/dist/loaders/html-loader.d.ts +4 -0
- package/dist/loaders/html-loader.d.ts.map +1 -1
- package/dist/loaders/html-loader.js +26 -2
- package/dist/loaders/html-loader.js.map +1 -1
- package/dist/loaders/index.d.ts.map +1 -1
- package/dist/loaders/index.js.map +1 -1
- package/dist/loaders/markdown-loader.d.ts.map +1 -1
- package/dist/loaders/markdown-loader.js +7 -3
- package/dist/loaders/markdown-loader.js.map +1 -1
- package/dist/loaders/text-loader.js +1 -1
- package/dist/loaders/text-loader.js.map +1 -1
- package/dist/loaders/web-loader.d.ts +29 -1
- package/dist/loaders/web-loader.d.ts.map +1 -1
- package/dist/loaders/web-loader.js +227 -92
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rag-builder.d.ts +10 -1
- package/dist/rag-builder.d.ts.map +1 -1
- package/dist/rag-builder.js +70 -7
- package/dist/rag-builder.js.map +1 -1
- package/dist/rag-pipeline.d.ts +1 -0
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +55 -19
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/rerankers/index.d.ts.map +1 -1
- package/dist/rerankers/index.js.map +1 -1
- package/dist/retrievers/chunk-indexer.d.ts +19 -0
- package/dist/retrievers/chunk-indexer.d.ts.map +1 -0
- package/dist/retrievers/chunk-indexer.js +8 -0
- package/dist/retrievers/chunk-indexer.js.map +1 -0
- package/dist/retrievers/create-retriever.d.ts +1 -0
- package/dist/retrievers/create-retriever.d.ts.map +1 -1
- package/dist/retrievers/create-retriever.js +1 -0
- package/dist/retrievers/create-retriever.js.map +1 -1
- package/dist/retrievers/hybrid-retriever.d.ts +8 -1
- package/dist/retrievers/hybrid-retriever.d.ts.map +1 -1
- package/dist/retrievers/hybrid-retriever.js +18 -5
- package/dist/retrievers/hybrid-retriever.js.map +1 -1
- package/dist/retrievers/index.d.ts +2 -0
- package/dist/retrievers/index.d.ts.map +1 -1
- package/dist/retrievers/index.js +1 -0
- package/dist/retrievers/index.js.map +1 -1
- package/dist/retrievers/mmr-retriever.d.ts +5 -0
- package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
- package/dist/retrievers/mmr-retriever.js +24 -2
- package/dist/retrievers/mmr-retriever.js.map +1 -1
- package/dist/retrievers/multi-query-retriever.d.ts +12 -1
- package/dist/retrievers/multi-query-retriever.d.ts.map +1 -1
- package/dist/retrievers/multi-query-retriever.js +46 -7
- package/dist/retrievers/multi-query-retriever.js.map +1 -1
- package/dist/retrievers/similarity-retriever.d.ts.map +1 -1
- package/dist/retrievers/similarity-retriever.js +2 -1
- package/dist/retrievers/similarity-retriever.js.map +1 -1
- package/dist/tools.d.ts +11 -2
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +39 -7
- package/dist/tools.js.map +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE1F,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAID;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,SAAS,EAAE,YAAY;IACjE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;gBAEhC,MAAM,EAAE,yBAAyB;IAO7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAM/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YA0B/E,aAAa;IAwB3B,OAAO,CAAC,mBAAmB;CAY5B"}
|
|
@@ -1,25 +1,64 @@
|
|
|
1
|
+
import { isChunkIndexer } from './chunk-indexer.js';
|
|
1
2
|
const DEFAULT_TOP_K = 10;
|
|
3
|
+
/**
|
|
4
|
+
* Expands the query into variants, retrieves for each in parallel and merges results
|
|
5
|
+
* (best score per chunk). The original query is always searched. If expansion fails the
|
|
6
|
+
* original query alone is used; if every retrieval fails the first error is thrown.
|
|
7
|
+
*/
|
|
2
8
|
export class MultiQueryRetriever {
|
|
3
9
|
baseRetriever;
|
|
4
10
|
expandQuery;
|
|
5
11
|
defaultTopK;
|
|
12
|
+
defaultMaxQueries;
|
|
6
13
|
constructor(config) {
|
|
7
14
|
this.baseRetriever = config.baseRetriever;
|
|
8
15
|
this.expandQuery = config.expandQuery;
|
|
9
16
|
this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
|
|
17
|
+
this.defaultMaxQueries = config.defaultMaxQueries;
|
|
10
18
|
}
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (!variants.includes(query)) {
|
|
15
|
-
variants.unshift(query);
|
|
19
|
+
indexChunk(chunk) {
|
|
20
|
+
if (isChunkIndexer(this.baseRetriever)) {
|
|
21
|
+
this.baseRetriever.indexChunk(chunk);
|
|
16
22
|
}
|
|
23
|
+
}
|
|
24
|
+
async retrieve(query, options) {
|
|
25
|
+
const variants = await this.buildVariants(query, options?.multiQueryCount);
|
|
17
26
|
const settled = await Promise.allSettled(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
|
|
18
|
-
const
|
|
19
|
-
|
|
27
|
+
const fulfilled = settled.filter((outcome) => outcome.status === 'fulfilled');
|
|
28
|
+
if (fulfilled.length === 0) {
|
|
29
|
+
const firstFailure = settled.find((outcome) => outcome.status === 'rejected');
|
|
30
|
+
throw firstFailure?.reason instanceof Error
|
|
31
|
+
? firstFailure.reason
|
|
32
|
+
: new Error(`MultiQueryRetriever: all retrievals failed: ${String(firstFailure?.reason)}`);
|
|
33
|
+
}
|
|
34
|
+
const merged = this.deduplicateAndMerge(fulfilled.flatMap((outcome) => outcome.value));
|
|
20
35
|
const topK = options?.topK ?? this.defaultTopK;
|
|
21
36
|
return merged.slice(0, topK);
|
|
22
37
|
}
|
|
38
|
+
async buildVariants(query, requestedMax) {
|
|
39
|
+
let expanded;
|
|
40
|
+
try {
|
|
41
|
+
expanded = await this.expandQuery(query);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
expanded = [];
|
|
45
|
+
}
|
|
46
|
+
const normalized = query.trim().toLowerCase();
|
|
47
|
+
const seen = new Set([normalized]);
|
|
48
|
+
const extra = [];
|
|
49
|
+
for (const variant of expanded) {
|
|
50
|
+
if (typeof variant !== 'string')
|
|
51
|
+
continue;
|
|
52
|
+
const trimmed = variant.trim();
|
|
53
|
+
const key = trimmed.toLowerCase();
|
|
54
|
+
if (!trimmed || seen.has(key))
|
|
55
|
+
continue;
|
|
56
|
+
seen.add(key);
|
|
57
|
+
extra.push(trimmed);
|
|
58
|
+
}
|
|
59
|
+
const max = requestedMax ?? this.defaultMaxQueries;
|
|
60
|
+
return [query, ...(max === undefined ? extra : extra.slice(0, max))];
|
|
61
|
+
}
|
|
23
62
|
deduplicateAndMerge(results) {
|
|
24
63
|
const best = new Map();
|
|
25
64
|
for (const result of results) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAwC,MAAM,oBAAoB,CAAC;AAU1F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IACpB,iBAAiB,CAAU;IAE5C,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpD,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QAE3E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,OAAO,EAAwD,EAAE,CAChE,OAAO,CAAC,MAAM,KAAK,WAAW,CACjC,CAAC;QACF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAC/B,CAAC,OAAO,EAAoC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAC7E,CAAC;YACF,MAAM,YAAY,EAAE,MAAM,YAAY,KAAK;gBACzC,CAAC,CAAC,YAAY,CAAC,MAAM;gBACrB,CAAC,CAAC,IAAI,KAAK,CAAC,+CAA+C,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,YAAqB;QAC9D,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,SAAS;YAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,GAAG,GAAG,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC;QACnD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resultSource } from './chunk-indexer.js';
|
|
1
2
|
const DEFAULT_TOP_K = 10;
|
|
2
3
|
const DEFAULT_THRESHOLD = 0.0;
|
|
3
4
|
export class SimilarityRetriever {
|
|
@@ -32,7 +33,7 @@ export class SimilarityRetriever {
|
|
|
32
33
|
documentId,
|
|
33
34
|
content: entry.content,
|
|
34
35
|
score: entry.score,
|
|
35
|
-
source: entry.sourceType,
|
|
36
|
+
source: resultSource(entry.metadata, entry.sourceType),
|
|
36
37
|
metadata: entry.metadata,
|
|
37
38
|
};
|
|
38
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
|
package/dist/tools.d.ts
CHANGED
|
@@ -17,7 +17,16 @@ declare const IngestParamsSchema: z.ZodObject<{
|
|
|
17
17
|
}, z.core.$strip>;
|
|
18
18
|
type IngestParams = z.infer<typeof IngestParamsSchema>;
|
|
19
19
|
export declare function createSearchTool(pipeline: RAGPipeline): RAGTool<SearchParams>;
|
|
20
|
-
export
|
|
21
|
-
|
|
20
|
+
export interface IngestToolOptions {
|
|
21
|
+
/**
|
|
22
|
+
* Restrict local sources to these directories (symlinks are resolved). When set, any
|
|
23
|
+
* path outside them is rejected. Strongly recommended when the tool is exposed to an LLM.
|
|
24
|
+
*/
|
|
25
|
+
allowedRoots?: string[];
|
|
26
|
+
/** Allow http(s) URLs as sources. Defaults to `true`. */
|
|
27
|
+
allowUrls?: boolean;
|
|
28
|
+
}
|
|
29
|
+
export declare function createIngestTool(pipeline: RAGPipeline, options?: IngestToolOptions): RAGTool<IngestParams>;
|
|
30
|
+
export declare function ragTools(pipeline: RAGPipeline, ingestOptions?: IngestToolOptions): [RAGTool<SearchParams>, RAGTool<IngestParams>];
|
|
22
31
|
export {};
|
|
23
32
|
//# sourceMappingURL=tools.d.ts.map
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAe7E;AAED,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,yDAAyD;IACzD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAiCD,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,YAAY,CAAC,CAgBvB;AAED,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,WAAW,EACrB,aAAa,CAAC,EAAE,iBAAiB,GAChC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAEhD"}
|
package/dist/tools.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
import { realpath } from 'node:fs/promises';
|
|
2
|
+
import { isAbsolute, relative, resolve } from 'node:path';
|
|
1
3
|
import { z } from 'zod';
|
|
2
4
|
const SearchParamsSchema = z.object({
|
|
3
|
-
query: z.string().min(1),
|
|
4
|
-
limit: z.number().int().positive().optional(),
|
|
5
|
-
threshold: z.number().min(0).max(1).optional(),
|
|
5
|
+
query: z.string().min(1).describe('Search query for the knowledge base'),
|
|
6
|
+
limit: z.number().int().positive().optional().describe('Maximum number of results to return'),
|
|
7
|
+
threshold: z.number().min(0).max(1).optional().describe('Minimum relevance score (0-1)'),
|
|
6
8
|
});
|
|
7
9
|
const IngestParamsSchema = z.object({
|
|
8
|
-
source: z.string().min(1),
|
|
10
|
+
source: z.string().min(1).describe('File path, directory or URL to ingest'),
|
|
9
11
|
});
|
|
10
12
|
export function createSearchTool(pipeline) {
|
|
11
13
|
return {
|
|
@@ -24,7 +26,36 @@ export function createSearchTool(pipeline) {
|
|
|
24
26
|
},
|
|
25
27
|
};
|
|
26
28
|
}
|
|
27
|
-
|
|
29
|
+
function isUrl(source) {
|
|
30
|
+
return /^[a-z][a-z0-9+.-]*:\/\//i.test(source);
|
|
31
|
+
}
|
|
32
|
+
async function canonicalPath(path) {
|
|
33
|
+
try {
|
|
34
|
+
return await realpath(path);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return resolve(path);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function assertAllowedSource(source, options) {
|
|
41
|
+
if (isUrl(source)) {
|
|
42
|
+
if (options.allowUrls === false) {
|
|
43
|
+
throw new Error('URL sources are not allowed');
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (!options.allowedRoots)
|
|
48
|
+
return;
|
|
49
|
+
const target = await canonicalPath(source);
|
|
50
|
+
for (const root of options.allowedRoots) {
|
|
51
|
+
const canonicalRoot = await canonicalPath(root);
|
|
52
|
+
const rel = relative(canonicalRoot, target);
|
|
53
|
+
if (rel === '' || (!rel.startsWith('..') && !isAbsolute(rel)))
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
throw new Error(`Source "${source}" is outside the allowed directories`);
|
|
57
|
+
}
|
|
58
|
+
export function createIngestTool(pipeline, options = {}) {
|
|
28
59
|
return {
|
|
29
60
|
name: 'rag_ingest',
|
|
30
61
|
description: 'Ingest documents into the knowledge base',
|
|
@@ -32,6 +63,7 @@ export function createIngestTool(pipeline) {
|
|
|
32
63
|
execute: async (params) => {
|
|
33
64
|
try {
|
|
34
65
|
const { source } = IngestParamsSchema.parse(params);
|
|
66
|
+
await assertAllowedSource(source, options);
|
|
35
67
|
const { documents, chunks } = await pipeline.ingest(source);
|
|
36
68
|
return { success: true, source, documents, chunks };
|
|
37
69
|
}
|
|
@@ -41,7 +73,7 @@ export function createIngestTool(pipeline) {
|
|
|
41
73
|
},
|
|
42
74
|
};
|
|
43
75
|
}
|
|
44
|
-
export function ragTools(pipeline) {
|
|
45
|
-
return [createSearchTool(pipeline), createIngestTool(pipeline)];
|
|
76
|
+
export function ragTools(pipeline, ingestOptions) {
|
|
77
|
+
return [createSearchTool(pipeline), createIngestTool(pipeline, ingestOptions)];
|
|
46
78
|
}
|
|
47
79
|
//# sourceMappingURL=tools.js.map
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC7F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACzF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;CAC5E,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAYD,SAAS,KAAK,CAAC,MAAc;IAC3B,OAAO,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAA0B;IAC3E,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO;IAElC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;IACxE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,sCAAsC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAqB,EACrB,UAA6B,EAAE;IAE/B,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,QAAqB,EACrB,aAAiC;IAEjC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACjF,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/rag",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Retrieval Augmented Generation for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
"dependencies": {
|
|
18
18
|
"nanoid": "^5.0.4",
|
|
19
19
|
"zod": "^4.3.6",
|
|
20
|
-
"@cogitator-ai/types": "0.
|
|
20
|
+
"@cogitator-ai/types": "0.23.0"
|
|
21
21
|
},
|
|
22
22
|
"peerDependencies": {
|
|
23
|
-
"@cogitator-ai/memory": "0.
|
|
23
|
+
"@cogitator-ai/memory": ">=0.7.0"
|
|
24
24
|
},
|
|
25
25
|
"peerDependenciesMeta": {
|
|
26
26
|
"@cogitator-ai/memory": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@types/pdf-parse": "^1.1.5",
|
|
38
38
|
"typescript": "^5.7.2",
|
|
39
39
|
"vitest": "^4.0.18",
|
|
40
|
-
"@cogitator-ai/memory": "0.
|
|
40
|
+
"@cogitator-ai/memory": "0.7.0"
|
|
41
41
|
},
|
|
42
42
|
"repository": {
|
|
43
43
|
"type": "git",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
},
|
|
51
51
|
"license": "MIT",
|
|
52
52
|
"scripts": {
|
|
53
|
-
"build": "tsc",
|
|
53
|
+
"build": "tsc && tsx ../../scripts/fix-esm-imports.ts .",
|
|
54
54
|
"dev": "tsc --watch",
|
|
55
55
|
"clean": "rm -rf dist",
|
|
56
56
|
"typecheck": "tsc --noEmit",
|