@cogitator-ai/rag 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -31
- package/dist/chunkers/create-chunker.js.map +1 -1
- package/dist/chunkers/index.d.ts.map +1 -1
- package/dist/chunkers/index.js.map +1 -1
- package/dist/chunkers/recursive-chunker.d.ts +2 -3
- package/dist/chunkers/recursive-chunker.d.ts.map +1 -1
- package/dist/chunkers/recursive-chunker.js +89 -118
- package/dist/chunkers/recursive-chunker.js.map +1 -1
- package/dist/chunkers/semantic-chunker.d.ts +2 -0
- package/dist/chunkers/semantic-chunker.d.ts.map +1 -1
- package/dist/chunkers/semantic-chunker.js +72 -38
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/loaders/csv-loader.d.ts.map +1 -1
- package/dist/loaders/csv-loader.js +12 -4
- package/dist/loaders/csv-loader.js.map +1 -1
- package/dist/loaders/html-loader.d.ts +4 -0
- package/dist/loaders/html-loader.d.ts.map +1 -1
- package/dist/loaders/html-loader.js +26 -2
- package/dist/loaders/html-loader.js.map +1 -1
- package/dist/loaders/index.d.ts.map +1 -1
- package/dist/loaders/index.js.map +1 -1
- package/dist/loaders/json-loader.d.ts +1 -0
- package/dist/loaders/json-loader.d.ts.map +1 -1
- package/dist/loaders/json-loader.js +15 -3
- package/dist/loaders/json-loader.js.map +1 -1
- package/dist/loaders/markdown-loader.d.ts.map +1 -1
- package/dist/loaders/markdown-loader.js +8 -4
- package/dist/loaders/markdown-loader.js.map +1 -1
- package/dist/loaders/pdf-loader.d.ts +1 -0
- package/dist/loaders/pdf-loader.d.ts.map +1 -1
- package/dist/loaders/pdf-loader.js +22 -3
- package/dist/loaders/pdf-loader.js.map +1 -1
- package/dist/loaders/text-loader.js +1 -1
- package/dist/loaders/text-loader.js.map +1 -1
- package/dist/loaders/web-loader.d.ts +33 -0
- package/dist/loaders/web-loader.d.ts.map +1 -1
- package/dist/loaders/web-loader.js +262 -8
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rag-builder.d.ts +10 -1
- package/dist/rag-builder.d.ts.map +1 -1
- package/dist/rag-builder.js +71 -6
- package/dist/rag-builder.js.map +1 -1
- package/dist/rag-pipeline.d.ts +2 -1
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +58 -23
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/rerankers/cohere-reranker.d.ts +2 -2
- package/dist/rerankers/cohere-reranker.d.ts.map +1 -1
- package/dist/rerankers/cohere-reranker.js +5 -0
- package/dist/rerankers/cohere-reranker.js.map +1 -1
- package/dist/rerankers/index.d.ts.map +1 -1
- package/dist/rerankers/index.js.map +1 -1
- package/dist/rerankers/llm-reranker.d.ts +1 -1
- package/dist/rerankers/llm-reranker.d.ts.map +1 -1
- package/dist/rerankers/llm-reranker.js +26 -13
- package/dist/rerankers/llm-reranker.js.map +1 -1
- package/dist/retrievers/chunk-indexer.d.ts +19 -0
- package/dist/retrievers/chunk-indexer.d.ts.map +1 -0
- package/dist/retrievers/chunk-indexer.js +8 -0
- package/dist/retrievers/chunk-indexer.js.map +1 -0
- package/dist/retrievers/create-retriever.d.ts +2 -0
- package/dist/retrievers/create-retriever.d.ts.map +1 -1
- package/dist/retrievers/create-retriever.js +6 -0
- package/dist/retrievers/create-retriever.js.map +1 -1
- package/dist/retrievers/hybrid-retriever.d.ts +8 -1
- package/dist/retrievers/hybrid-retriever.d.ts.map +1 -1
- package/dist/retrievers/hybrid-retriever.js +18 -5
- package/dist/retrievers/hybrid-retriever.js.map +1 -1
- package/dist/retrievers/index.d.ts +2 -0
- package/dist/retrievers/index.d.ts.map +1 -1
- package/dist/retrievers/index.js +1 -0
- package/dist/retrievers/index.js.map +1 -1
- package/dist/retrievers/mmr-retriever.d.ts +5 -0
- package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
- package/dist/retrievers/mmr-retriever.js +34 -8
- package/dist/retrievers/mmr-retriever.js.map +1 -1
- package/dist/retrievers/multi-query-retriever.d.ts +14 -1
- package/dist/retrievers/multi-query-retriever.d.ts.map +1 -1
- package/dist/retrievers/multi-query-retriever.js +50 -6
- package/dist/retrievers/multi-query-retriever.js.map +1 -1
- package/dist/retrievers/similarity-retriever.d.ts.map +1 -1
- package/dist/retrievers/similarity-retriever.js +2 -1
- package/dist/retrievers/similarity-retriever.js.map +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +16 -2
- package/dist/schema.js.map +1 -1
- package/dist/tools.d.ts +11 -2
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +43 -9
- package/dist/tools.js.map +1 -1
- package/package.json +5 -5
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,UAA6B;QACvD,MAAM,OAAO,GAAG,UAAU;aACvB,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAC7C,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAClD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,aAAa,CAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QAC/B,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YAC1C,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY,EACZ,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBACD,IAAI,cAAc,KAAK,CAAC,QAAQ;oBAAE,cAAc,GAAG,CAAC,CAAC;gBAErD,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,13 +1,26 @@
|
|
|
1
1
|
import type { Retriever, RetrievalConfig, RetrievalResult } from '@cogitator-ai/types';
|
|
2
|
+
import { type ChunkIndexer, type IndexedChunk } from './chunk-indexer.js';
|
|
2
3
|
export interface MultiQueryRetrieverConfig {
|
|
3
4
|
baseRetriever: Retriever;
|
|
4
5
|
expandQuery: (query: string) => Promise<string[]>;
|
|
6
|
+
defaultTopK?: number;
|
|
7
|
+
/** Maximum number of generated query variants used in addition to the original query. */
|
|
8
|
+
defaultMaxQueries?: number;
|
|
5
9
|
}
|
|
6
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Expands the query into variants, retrieves for each in parallel and merges results
|
|
12
|
+
* (best score per chunk). The original query is always searched. If expansion fails the
|
|
13
|
+
* original query alone is used; if every retrieval fails the first error is thrown.
|
|
14
|
+
*/
|
|
15
|
+
export declare class MultiQueryRetriever implements Retriever, ChunkIndexer {
|
|
7
16
|
private readonly baseRetriever;
|
|
8
17
|
private readonly expandQuery;
|
|
18
|
+
private readonly defaultTopK;
|
|
19
|
+
private readonly defaultMaxQueries?;
|
|
9
20
|
constructor(config: MultiQueryRetrieverConfig);
|
|
21
|
+
indexChunk(chunk: IndexedChunk): void;
|
|
10
22
|
retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
|
|
23
|
+
private buildVariants;
|
|
11
24
|
private deduplicateAndMerge;
|
|
12
25
|
}
|
|
13
26
|
//# sourceMappingURL=multi-query-retriever.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE1F,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAID;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,SAAS,EAAE,YAAY;IACjE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;gBAEhC,MAAM,EAAE,yBAAyB;IAO7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAM/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YA0B/E,aAAa;IAwB3B,OAAO,CAAC,mBAAmB;CAY5B"}
|
|
@@ -1,20 +1,64 @@
|
|
|
1
|
+
import { isChunkIndexer } from './chunk-indexer.js';
|
|
1
2
|
const DEFAULT_TOP_K = 10;
|
|
3
|
+
/**
|
|
4
|
+
* Expands the query into variants, retrieves for each in parallel and merges results
|
|
5
|
+
* (best score per chunk). The original query is always searched. If expansion fails the
|
|
6
|
+
* original query alone is used; if every retrieval fails the first error is thrown.
|
|
7
|
+
*/
|
|
2
8
|
export class MultiQueryRetriever {
|
|
3
9
|
baseRetriever;
|
|
4
10
|
expandQuery;
|
|
11
|
+
defaultTopK;
|
|
12
|
+
defaultMaxQueries;
|
|
5
13
|
constructor(config) {
|
|
6
14
|
this.baseRetriever = config.baseRetriever;
|
|
7
15
|
this.expandQuery = config.expandQuery;
|
|
16
|
+
this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
|
|
17
|
+
this.defaultMaxQueries = config.defaultMaxQueries;
|
|
18
|
+
}
|
|
19
|
+
indexChunk(chunk) {
|
|
20
|
+
if (isChunkIndexer(this.baseRetriever)) {
|
|
21
|
+
this.baseRetriever.indexChunk(chunk);
|
|
22
|
+
}
|
|
8
23
|
}
|
|
9
24
|
async retrieve(query, options) {
|
|
10
|
-
const variants = await this.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
25
|
+
const variants = await this.buildVariants(query, options?.multiQueryCount);
|
|
26
|
+
const settled = await Promise.allSettled(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
|
|
27
|
+
const fulfilled = settled.filter((outcome) => outcome.status === 'fulfilled');
|
|
28
|
+
if (fulfilled.length === 0) {
|
|
29
|
+
const firstFailure = settled.find((outcome) => outcome.status === 'rejected');
|
|
30
|
+
throw firstFailure?.reason instanceof Error
|
|
31
|
+
? firstFailure.reason
|
|
32
|
+
: new Error(`MultiQueryRetriever: all retrievals failed: ${String(firstFailure?.reason)}`);
|
|
33
|
+
}
|
|
34
|
+
const merged = this.deduplicateAndMerge(fulfilled.flatMap((outcome) => outcome.value));
|
|
35
|
+
const topK = options?.topK ?? this.defaultTopK;
|
|
16
36
|
return merged.slice(0, topK);
|
|
17
37
|
}
|
|
38
|
+
async buildVariants(query, requestedMax) {
|
|
39
|
+
let expanded;
|
|
40
|
+
try {
|
|
41
|
+
expanded = await this.expandQuery(query);
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
expanded = [];
|
|
45
|
+
}
|
|
46
|
+
const normalized = query.trim().toLowerCase();
|
|
47
|
+
const seen = new Set([normalized]);
|
|
48
|
+
const extra = [];
|
|
49
|
+
for (const variant of expanded) {
|
|
50
|
+
if (typeof variant !== 'string')
|
|
51
|
+
continue;
|
|
52
|
+
const trimmed = variant.trim();
|
|
53
|
+
const key = trimmed.toLowerCase();
|
|
54
|
+
if (!trimmed || seen.has(key))
|
|
55
|
+
continue;
|
|
56
|
+
seen.add(key);
|
|
57
|
+
extra.push(trimmed);
|
|
58
|
+
}
|
|
59
|
+
const max = requestedMax ?? this.defaultMaxQueries;
|
|
60
|
+
return [query, ...(max === undefined ? extra : extra.slice(0, max))];
|
|
61
|
+
}
|
|
18
62
|
deduplicateAndMerge(results) {
|
|
19
63
|
const best = new Map();
|
|
20
64
|
for (const result of results) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAwC,MAAM,oBAAoB,CAAC;AAU1F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IACpB,iBAAiB,CAAU;IAE5C,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpD,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QAE3E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,OAAO,EAAwD,EAAE,CAChE,OAAO,CAAC,MAAM,KAAK,WAAW,CACjC,CAAC;QACF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAC/B,CAAC,OAAO,EAAoC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAC7E,CAAC;YACF,MAAM,YAAY,EAAE,MAAM,YAAY,KAAK;gBACzC,CAAC,CAAC,YAAY,CAAC,MAAM;gBACrB,CAAC,CAAC,IAAI,KAAK,CAAC,+CAA+C,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,YAAqB;QAC9D,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,SAAS;YAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,GAAG,GAAG,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC;QACnD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resultSource } from './chunk-indexer.js';
|
|
1
2
|
const DEFAULT_TOP_K = 10;
|
|
2
3
|
const DEFAULT_THRESHOLD = 0.0;
|
|
3
4
|
export class SimilarityRetriever {
|
|
@@ -32,7 +33,7 @@ export class SimilarityRetriever {
|
|
|
32
33
|
documentId,
|
|
33
34
|
content: entry.content,
|
|
34
35
|
score: entry.score,
|
|
35
|
-
source: entry.sourceType,
|
|
36
|
+
source: resultSource(entry.metadata, entry.sourceType),
|
|
36
37
|
metadata: entry.metadata,
|
|
37
38
|
};
|
|
38
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;iBAU7B,CAAC;AAEL,eAAO,MAAM,uBAAuB;;;;;EAAyD,CAAC;AAE9F,eAAO,MAAM,qBAAqB;;;;;;;;;;;iBAe9B,CAAC;AAIL,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AACvE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
|
package/dist/schema.js
CHANGED
|
@@ -1,18 +1,32 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
export const ChunkingStrategySchema = z.enum(['fixed', 'recursive', 'semantic']);
|
|
3
|
-
export const ChunkingConfigSchema = z
|
|
3
|
+
export const ChunkingConfigSchema = z
|
|
4
|
+
.object({
|
|
4
5
|
strategy: ChunkingStrategySchema,
|
|
5
6
|
chunkSize: z.number().int().positive(),
|
|
6
7
|
chunkOverlap: z.number().int().nonnegative().default(0),
|
|
7
8
|
separators: z.array(z.string()).optional(),
|
|
9
|
+
})
|
|
10
|
+
.refine((d) => d.chunkOverlap < d.chunkSize, {
|
|
11
|
+
message: 'chunkOverlap must be less than chunkSize',
|
|
12
|
+
path: ['chunkOverlap'],
|
|
8
13
|
});
|
|
9
14
|
export const RetrievalStrategySchema = z.enum(['similarity', 'mmr', 'hybrid', 'multi-query']);
|
|
10
|
-
export const RetrievalConfigSchema = z
|
|
15
|
+
export const RetrievalConfigSchema = z
|
|
16
|
+
.object({
|
|
11
17
|
strategy: RetrievalStrategySchema.default('similarity'),
|
|
12
18
|
topK: z.number().int().positive().default(10),
|
|
13
19
|
threshold: z.number().min(0).max(1).default(0.0),
|
|
14
20
|
mmrLambda: z.number().min(0).max(1).optional(),
|
|
15
21
|
multiQueryCount: z.number().int().positive().optional(),
|
|
22
|
+
})
|
|
23
|
+
.refine((d) => d.strategy !== 'mmr' || d.mmrLambda !== undefined, {
|
|
24
|
+
message: 'mmrLambda is required when strategy is "mmr"',
|
|
25
|
+
path: ['mmrLambda'],
|
|
26
|
+
})
|
|
27
|
+
.refine((d) => d.strategy !== 'multi-query' || d.multiQueryCount !== undefined, {
|
|
28
|
+
message: 'multiQueryCount is required when strategy is "multi-query"',
|
|
29
|
+
path: ['multiQueryCount'],
|
|
16
30
|
});
|
|
17
31
|
const retrievalDefaults = RetrievalConfigSchema.parse({});
|
|
18
32
|
export const RerankingConfigSchema = z.object({
|
package/dist/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,QAAQ,EAAE,sBAAsB;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,SAAS,EAAE;IAC3C,OAAO,EAAE,0CAA0C;IACnD,IAAI,EAAE,CAAC,cAAc,CAAC;CACvB,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,QAAQ,EAAE,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC;IACvD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE;IAChE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,WAAW,CAAC;CACpB,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS,EAAE;IAC9E,OAAO,EAAE,4DAA4D;IACrE,IAAI,EAAE,CAAC,iBAAiB,CAAC;CAC1B,CAAC,CAAC;AAEL,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAC3D,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC"}
|
package/dist/tools.d.ts
CHANGED
|
@@ -17,7 +17,16 @@ declare const IngestParamsSchema: z.ZodObject<{
|
|
|
17
17
|
}, z.core.$strip>;
|
|
18
18
|
type IngestParams = z.infer<typeof IngestParamsSchema>;
|
|
19
19
|
export declare function createSearchTool(pipeline: RAGPipeline): RAGTool<SearchParams>;
|
|
20
|
-
export
|
|
21
|
-
|
|
20
|
+
export interface IngestToolOptions {
|
|
21
|
+
/**
|
|
22
|
+
* Restrict local sources to these directories (symlinks are resolved). When set, any
|
|
23
|
+
* path outside them is rejected. Strongly recommended when the tool is exposed to an LLM.
|
|
24
|
+
*/
|
|
25
|
+
allowedRoots?: string[];
|
|
26
|
+
/** Allow http(s) URLs as sources. Defaults to `true`. */
|
|
27
|
+
allowUrls?: boolean;
|
|
28
|
+
}
|
|
29
|
+
export declare function createIngestTool(pipeline: RAGPipeline, options?: IngestToolOptions): RAGTool<IngestParams>;
|
|
30
|
+
export declare function ragTools(pipeline: RAGPipeline, ingestOptions?: IngestToolOptions): [RAGTool<SearchParams>, RAGTool<IngestParams>];
|
|
22
31
|
export {};
|
|
23
32
|
//# sourceMappingURL=tools.d.ts.map
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAe7E;AAED,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,yDAAyD;IACzD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAiCD,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,YAAY,CAAC,CAgBvB;AAED,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,WAAW,EACrB,aAAa,CAAC,EAAE,iBAAiB,GAChC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAEhD"}
|
package/dist/tools.js
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
|
+
import { realpath } from 'node:fs/promises';
|
|
2
|
+
import { isAbsolute, relative, resolve } from 'node:path';
|
|
1
3
|
import { z } from 'zod';
|
|
2
4
|
const SearchParamsSchema = z.object({
|
|
3
|
-
query: z.string(),
|
|
4
|
-
limit: z.number().int().positive().optional(),
|
|
5
|
-
threshold: z.number().min(0).max(1).optional(),
|
|
5
|
+
query: z.string().min(1).describe('Search query for the knowledge base'),
|
|
6
|
+
limit: z.number().int().positive().optional().describe('Maximum number of results to return'),
|
|
7
|
+
threshold: z.number().min(0).max(1).optional().describe('Minimum relevance score (0-1)'),
|
|
6
8
|
});
|
|
7
9
|
const IngestParamsSchema = z.object({
|
|
8
|
-
source: z.string(),
|
|
10
|
+
source: z.string().min(1).describe('File path, directory or URL to ingest'),
|
|
9
11
|
});
|
|
10
12
|
export function createSearchTool(pipeline) {
|
|
11
13
|
return {
|
|
12
14
|
name: 'rag_search',
|
|
13
15
|
description: 'Search the knowledge base using semantic search',
|
|
14
16
|
parameters: SearchParamsSchema,
|
|
15
|
-
execute: async (
|
|
17
|
+
execute: async (params) => {
|
|
16
18
|
try {
|
|
19
|
+
const { query, limit, threshold } = SearchParamsSchema.parse(params);
|
|
17
20
|
const results = await pipeline.query(query, { topK: limit, threshold });
|
|
18
21
|
return { success: true, query, results, count: results.length };
|
|
19
22
|
}
|
|
@@ -23,13 +26,44 @@ export function createSearchTool(pipeline) {
|
|
|
23
26
|
},
|
|
24
27
|
};
|
|
25
28
|
}
|
|
26
|
-
|
|
29
|
+
function isUrl(source) {
|
|
30
|
+
return /^[a-z][a-z0-9+.-]*:\/\//i.test(source);
|
|
31
|
+
}
|
|
32
|
+
async function canonicalPath(path) {
|
|
33
|
+
try {
|
|
34
|
+
return await realpath(path);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return resolve(path);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function assertAllowedSource(source, options) {
|
|
41
|
+
if (isUrl(source)) {
|
|
42
|
+
if (options.allowUrls === false) {
|
|
43
|
+
throw new Error('URL sources are not allowed');
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (!options.allowedRoots)
|
|
48
|
+
return;
|
|
49
|
+
const target = await canonicalPath(source);
|
|
50
|
+
for (const root of options.allowedRoots) {
|
|
51
|
+
const canonicalRoot = await canonicalPath(root);
|
|
52
|
+
const rel = relative(canonicalRoot, target);
|
|
53
|
+
if (rel === '' || (!rel.startsWith('..') && !isAbsolute(rel)))
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
throw new Error(`Source "${source}" is outside the allowed directories`);
|
|
57
|
+
}
|
|
58
|
+
export function createIngestTool(pipeline, options = {}) {
|
|
27
59
|
return {
|
|
28
60
|
name: 'rag_ingest',
|
|
29
61
|
description: 'Ingest documents into the knowledge base',
|
|
30
62
|
parameters: IngestParamsSchema,
|
|
31
|
-
execute: async (
|
|
63
|
+
execute: async (params) => {
|
|
32
64
|
try {
|
|
65
|
+
const { source } = IngestParamsSchema.parse(params);
|
|
66
|
+
await assertAllowedSource(source, options);
|
|
33
67
|
const { documents, chunks } = await pipeline.ingest(source);
|
|
34
68
|
return { success: true, source, documents, chunks };
|
|
35
69
|
}
|
|
@@ -39,7 +73,7 @@ export function createIngestTool(pipeline) {
|
|
|
39
73
|
},
|
|
40
74
|
};
|
|
41
75
|
}
|
|
42
|
-
export function ragTools(pipeline) {
|
|
43
|
-
return [createSearchTool(pipeline), createIngestTool(pipeline)];
|
|
76
|
+
export function ragTools(pipeline, ingestOptions) {
|
|
77
|
+
return [createSearchTool(pipeline), createIngestTool(pipeline, ingestOptions)];
|
|
44
78
|
}
|
|
45
79
|
//# sourceMappingURL=tools.js.map
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC7F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACzF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;CAC5E,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAYD,SAAS,KAAK,CAAC,MAAc;IAC3B,OAAO,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAA0B;IAC3E,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO;IAElC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;IACxE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,sCAAsC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAqB,EACrB,UAA6B,EAAE;IAE/B,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,QAAqB,EACrB,aAAiC;IAEjC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACjF,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/rag",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Retrieval Augmented Generation for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -17,10 +17,10 @@
|
|
|
17
17
|
"dependencies": {
|
|
18
18
|
"nanoid": "^5.0.4",
|
|
19
19
|
"zod": "^4.3.6",
|
|
20
|
-
"@cogitator-ai/types": "0.
|
|
20
|
+
"@cogitator-ai/types": "0.23.0"
|
|
21
21
|
},
|
|
22
22
|
"peerDependencies": {
|
|
23
|
-
"@cogitator-ai/memory": "0.
|
|
23
|
+
"@cogitator-ai/memory": ">=0.7.0"
|
|
24
24
|
},
|
|
25
25
|
"peerDependenciesMeta": {
|
|
26
26
|
"@cogitator-ai/memory": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"@types/pdf-parse": "^1.1.5",
|
|
38
38
|
"typescript": "^5.7.2",
|
|
39
39
|
"vitest": "^4.0.18",
|
|
40
|
-
"@cogitator-ai/memory": "0.
|
|
40
|
+
"@cogitator-ai/memory": "0.7.0"
|
|
41
41
|
},
|
|
42
42
|
"repository": {
|
|
43
43
|
"type": "git",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
},
|
|
51
51
|
"license": "MIT",
|
|
52
52
|
"scripts": {
|
|
53
|
-
"build": "tsc",
|
|
53
|
+
"build": "tsc && tsx ../../scripts/fix-esm-imports.ts .",
|
|
54
54
|
"dev": "tsc --watch",
|
|
55
55
|
"clean": "rm -rf dist",
|
|
56
56
|
"typecheck": "tsc --noEmit",
|