@cogitator-ai/rag 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/chunkers/recursive-chunker.d.ts +2 -2
- package/dist/chunkers/recursive-chunker.d.ts.map +1 -1
- package/dist/chunkers/recursive-chunker.js +49 -88
- package/dist/chunkers/recursive-chunker.js.map +1 -1
- package/dist/chunkers/semantic-chunker.d.ts +2 -0
- package/dist/chunkers/semantic-chunker.d.ts.map +1 -1
- package/dist/chunkers/semantic-chunker.js +68 -37
- package/dist/chunkers/semantic-chunker.js.map +1 -1
- package/dist/loaders/csv-loader.d.ts.map +1 -1
- package/dist/loaders/csv-loader.js +3 -0
- package/dist/loaders/csv-loader.js.map +1 -1
- package/dist/loaders/json-loader.d.ts +1 -0
- package/dist/loaders/json-loader.d.ts.map +1 -1
- package/dist/loaders/json-loader.js +15 -3
- package/dist/loaders/json-loader.js.map +1 -1
- package/dist/loaders/markdown-loader.js +1 -1
- package/dist/loaders/markdown-loader.js.map +1 -1
- package/dist/loaders/pdf-loader.d.ts +1 -0
- package/dist/loaders/pdf-loader.d.ts.map +1 -1
- package/dist/loaders/pdf-loader.js +22 -3
- package/dist/loaders/pdf-loader.js.map +1 -1
- package/dist/loaders/web-loader.d.ts +5 -0
- package/dist/loaders/web-loader.d.ts.map +1 -1
- package/dist/loaders/web-loader.js +127 -8
- package/dist/loaders/web-loader.js.map +1 -1
- package/dist/rag-builder.d.ts.map +1 -1
- package/dist/rag-builder.js +3 -1
- package/dist/rag-builder.js.map +1 -1
- package/dist/rag-pipeline.d.ts +1 -1
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +5 -6
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/rerankers/cohere-reranker.d.ts +2 -2
- package/dist/rerankers/cohere-reranker.d.ts.map +1 -1
- package/dist/rerankers/cohere-reranker.js +5 -0
- package/dist/rerankers/cohere-reranker.js.map +1 -1
- package/dist/rerankers/llm-reranker.d.ts +1 -1
- package/dist/rerankers/llm-reranker.d.ts.map +1 -1
- package/dist/rerankers/llm-reranker.js +26 -13
- package/dist/rerankers/llm-reranker.js.map +1 -1
- package/dist/retrievers/create-retriever.d.ts +1 -0
- package/dist/retrievers/create-retriever.d.ts.map +1 -1
- package/dist/retrievers/create-retriever.js +5 -0
- package/dist/retrievers/create-retriever.js.map +1 -1
- package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
- package/dist/retrievers/mmr-retriever.js +11 -7
- package/dist/retrievers/mmr-retriever.js.map +1 -1
- package/dist/retrievers/multi-query-retriever.d.ts +2 -0
- package/dist/retrievers/multi-query-retriever.d.ts.map +1 -1
- package/dist/retrievers/multi-query-retriever.js +10 -5
- package/dist/retrievers/multi-query-retriever.js.map +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +16 -2
- package/dist/schema.js.map +1 -1
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +6 -4
- package/dist/tools.js.map +1 -1
- package/package.json +16 -16
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,
|
|
1
|
+
{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY,EACZ,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBACD,IAAI,cAAc,KAAK,CAAC,QAAQ;oBAAE,cAAc,GAAG,CAAC,CAAC;gBAErD,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -2,10 +2,12 @@ import type { Retriever, RetrievalConfig, RetrievalResult } from '@cogitator-ai/
|
|
|
2
2
|
export interface MultiQueryRetrieverConfig {
|
|
3
3
|
baseRetriever: Retriever;
|
|
4
4
|
expandQuery: (query: string) => Promise<string[]>;
|
|
5
|
+
defaultTopK?: number;
|
|
5
6
|
}
|
|
6
7
|
export declare class MultiQueryRetriever implements Retriever {
|
|
7
8
|
private readonly baseRetriever;
|
|
8
9
|
private readonly expandQuery;
|
|
10
|
+
private readonly defaultTopK;
|
|
9
11
|
constructor(config: MultiQueryRetrieverConfig);
|
|
10
12
|
retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
|
|
11
13
|
private deduplicateAndMerge;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,MAAM,EAAE,yBAAyB;IAMvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAsB7F,OAAO,CAAC,mBAAmB;CAY5B"}
|
|
@@ -2,17 +2,22 @@ const DEFAULT_TOP_K = 10;
|
|
|
2
2
|
export class MultiQueryRetriever {
|
|
3
3
|
baseRetriever;
|
|
4
4
|
expandQuery;
|
|
5
|
+
defaultTopK;
|
|
5
6
|
constructor(config) {
|
|
6
7
|
this.baseRetriever = config.baseRetriever;
|
|
7
8
|
this.expandQuery = config.expandQuery;
|
|
9
|
+
this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
|
|
8
10
|
}
|
|
9
11
|
async retrieve(query, options) {
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
const expanded = await this.expandQuery(query);
|
|
13
|
+
const variants = expanded.length > 0 ? expanded : [query];
|
|
14
|
+
if (!variants.includes(query)) {
|
|
15
|
+
variants.unshift(query);
|
|
16
|
+
}
|
|
17
|
+
const settled = await Promise.allSettled(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
|
|
18
|
+
const allResults = settled.flatMap((outcome) => outcome.status === 'fulfilled' ? outcome.value : []);
|
|
14
19
|
const merged = this.deduplicateAndMerge(allResults.flat());
|
|
15
|
-
const topK = options?.topK ??
|
|
20
|
+
const topK = options?.topK ?? this.defaultTopK;
|
|
16
21
|
return merged.slice(0, topK);
|
|
17
22
|
}
|
|
18
23
|
deduplicateAndMerge(results) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAQA,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IAErC,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAE1D,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7C,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;iBAU7B,CAAC;AAEL,eAAO,MAAM,uBAAuB;;;;;EAAyD,CAAC;AAE9F,eAAO,MAAM,qBAAqB;;;;;;;;;;;iBAe9B,CAAC;AAIL,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AACvE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
|
package/dist/schema.js
CHANGED
|
@@ -1,18 +1,32 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
export const ChunkingStrategySchema = z.enum(['fixed', 'recursive', 'semantic']);
|
|
3
|
-
export const ChunkingConfigSchema = z
|
|
3
|
+
export const ChunkingConfigSchema = z
|
|
4
|
+
.object({
|
|
4
5
|
strategy: ChunkingStrategySchema,
|
|
5
6
|
chunkSize: z.number().int().positive(),
|
|
6
7
|
chunkOverlap: z.number().int().nonnegative().default(0),
|
|
7
8
|
separators: z.array(z.string()).optional(),
|
|
9
|
+
})
|
|
10
|
+
.refine((d) => d.chunkOverlap < d.chunkSize, {
|
|
11
|
+
message: 'chunkOverlap must be less than chunkSize',
|
|
12
|
+
path: ['chunkOverlap'],
|
|
8
13
|
});
|
|
9
14
|
export const RetrievalStrategySchema = z.enum(['similarity', 'mmr', 'hybrid', 'multi-query']);
|
|
10
|
-
export const RetrievalConfigSchema = z
|
|
15
|
+
export const RetrievalConfigSchema = z
|
|
16
|
+
.object({
|
|
11
17
|
strategy: RetrievalStrategySchema.default('similarity'),
|
|
12
18
|
topK: z.number().int().positive().default(10),
|
|
13
19
|
threshold: z.number().min(0).max(1).default(0.0),
|
|
14
20
|
mmrLambda: z.number().min(0).max(1).optional(),
|
|
15
21
|
multiQueryCount: z.number().int().positive().optional(),
|
|
22
|
+
})
|
|
23
|
+
.refine((d) => d.strategy !== 'mmr' || d.mmrLambda !== undefined, {
|
|
24
|
+
message: 'mmrLambda is required when strategy is "mmr"',
|
|
25
|
+
path: ['mmrLambda'],
|
|
26
|
+
})
|
|
27
|
+
.refine((d) => d.strategy !== 'multi-query' || d.multiQueryCount !== undefined, {
|
|
28
|
+
message: 'multiQueryCount is required when strategy is "multi-query"',
|
|
29
|
+
path: ['multiQueryCount'],
|
|
16
30
|
});
|
|
17
31
|
const retrievalDefaults = RetrievalConfigSchema.parse({});
|
|
18
32
|
export const RerankingConfigSchema = z.object({
|
package/dist/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,QAAQ,EAAE,sBAAsB;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,SAAS,EAAE;IAC3C,OAAO,EAAE,0CAA0C;IACnD,IAAI,EAAE,CAAC,cAAc,CAAC;CACvB,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,QAAQ,EAAE,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC;IACvD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE;IAChE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,WAAW,CAAC;CACpB,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS,EAAE;IAC9E,OAAO,EAAE,4DAA4D;IACrE,IAAI,EAAE,CAAC,iBAAiB,CAAC;CAC1B,CAAC,CAAC;AAEL,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAC3D,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC"}
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAe7E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAe7E;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,WAAW,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAE9F"}
|
package/dist/tools.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
const SearchParamsSchema = z.object({
|
|
3
|
-
query: z.string(),
|
|
3
|
+
query: z.string().min(1),
|
|
4
4
|
limit: z.number().int().positive().optional(),
|
|
5
5
|
threshold: z.number().min(0).max(1).optional(),
|
|
6
6
|
});
|
|
7
7
|
const IngestParamsSchema = z.object({
|
|
8
|
-
source: z.string(),
|
|
8
|
+
source: z.string().min(1),
|
|
9
9
|
});
|
|
10
10
|
export function createSearchTool(pipeline) {
|
|
11
11
|
return {
|
|
12
12
|
name: 'rag_search',
|
|
13
13
|
description: 'Search the knowledge base using semantic search',
|
|
14
14
|
parameters: SearchParamsSchema,
|
|
15
|
-
execute: async (
|
|
15
|
+
execute: async (params) => {
|
|
16
16
|
try {
|
|
17
|
+
const { query, limit, threshold } = SearchParamsSchema.parse(params);
|
|
17
18
|
const results = await pipeline.query(query, { topK: limit, threshold });
|
|
18
19
|
return { success: true, query, results, count: results.length };
|
|
19
20
|
}
|
|
@@ -28,8 +29,9 @@ export function createIngestTool(pipeline) {
|
|
|
28
29
|
name: 'rag_ingest',
|
|
29
30
|
description: 'Ingest documents into the knowledge base',
|
|
30
31
|
parameters: IngestParamsSchema,
|
|
31
|
-
execute: async (
|
|
32
|
+
execute: async (params) => {
|
|
32
33
|
try {
|
|
34
|
+
const { source } = IngestParamsSchema.parse(params);
|
|
33
35
|
const { documents, chunks } = await pipeline.ingest(source);
|
|
34
36
|
return { success: true, source, documents, chunks };
|
|
35
37
|
}
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAC1B,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,QAAqB;IAC5C,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAClE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/rag",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.10",
|
|
4
4
|
"description": "Retrieval Augmented Generation for Cogitator AI agents",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -14,21 +14,13 @@
|
|
|
14
14
|
"files": [
|
|
15
15
|
"dist"
|
|
16
16
|
],
|
|
17
|
-
"scripts": {
|
|
18
|
-
"build": "tsc",
|
|
19
|
-
"dev": "tsc --watch",
|
|
20
|
-
"clean": "rm -rf dist",
|
|
21
|
-
"typecheck": "tsc --noEmit",
|
|
22
|
-
"test": "vitest run",
|
|
23
|
-
"test:watch": "vitest"
|
|
24
|
-
},
|
|
25
17
|
"dependencies": {
|
|
26
|
-
"@cogitator-ai/types": "workspace:*",
|
|
27
18
|
"nanoid": "^5.0.4",
|
|
28
|
-
"zod": "^4.3.6"
|
|
19
|
+
"zod": "^4.3.6",
|
|
20
|
+
"@cogitator-ai/types": "0.22.3"
|
|
29
21
|
},
|
|
30
22
|
"peerDependencies": {
|
|
31
|
-
"@cogitator-ai/memory": "
|
|
23
|
+
"@cogitator-ai/memory": "0.6.22"
|
|
32
24
|
},
|
|
33
25
|
"peerDependenciesMeta": {
|
|
34
26
|
"@cogitator-ai/memory": {
|
|
@@ -41,11 +33,11 @@
|
|
|
41
33
|
"pdf-parse": "^1.1.1"
|
|
42
34
|
},
|
|
43
35
|
"devDependencies": {
|
|
44
|
-
"@cogitator-ai/memory": "workspace:*",
|
|
45
36
|
"@types/papaparse": "^5.3.15",
|
|
46
37
|
"@types/pdf-parse": "^1.1.5",
|
|
47
38
|
"typescript": "^5.7.2",
|
|
48
|
-
"vitest": "^4.0.18"
|
|
39
|
+
"vitest": "^4.0.18",
|
|
40
|
+
"@cogitator-ai/memory": "0.6.22"
|
|
49
41
|
},
|
|
50
42
|
"repository": {
|
|
51
43
|
"type": "git",
|
|
@@ -56,5 +48,13 @@
|
|
|
56
48
|
"access": "public",
|
|
57
49
|
"registry": "https://npm.pkg.github.com"
|
|
58
50
|
},
|
|
59
|
-
"license": "MIT"
|
|
60
|
-
|
|
51
|
+
"license": "MIT",
|
|
52
|
+
"scripts": {
|
|
53
|
+
"build": "tsc",
|
|
54
|
+
"dev": "tsc --watch",
|
|
55
|
+
"clean": "rm -rf dist",
|
|
56
|
+
"typecheck": "tsc --noEmit",
|
|
57
|
+
"test": "vitest run",
|
|
58
|
+
"test:watch": "vitest"
|
|
59
|
+
}
|
|
60
|
+
}
|