@cogitator-ai/rag 0.1.9 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +66 -31
  2. package/dist/chunkers/create-chunker.js.map +1 -1
  3. package/dist/chunkers/index.d.ts.map +1 -1
  4. package/dist/chunkers/index.js.map +1 -1
  5. package/dist/chunkers/recursive-chunker.d.ts +2 -3
  6. package/dist/chunkers/recursive-chunker.d.ts.map +1 -1
  7. package/dist/chunkers/recursive-chunker.js +89 -118
  8. package/dist/chunkers/recursive-chunker.js.map +1 -1
  9. package/dist/chunkers/semantic-chunker.d.ts +2 -0
  10. package/dist/chunkers/semantic-chunker.d.ts.map +1 -1
  11. package/dist/chunkers/semantic-chunker.js +72 -38
  12. package/dist/chunkers/semantic-chunker.js.map +1 -1
  13. package/dist/index.d.ts +2 -2
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +4 -2
  16. package/dist/index.js.map +1 -1
  17. package/dist/loaders/csv-loader.d.ts.map +1 -1
  18. package/dist/loaders/csv-loader.js +12 -4
  19. package/dist/loaders/csv-loader.js.map +1 -1
  20. package/dist/loaders/html-loader.d.ts +4 -0
  21. package/dist/loaders/html-loader.d.ts.map +1 -1
  22. package/dist/loaders/html-loader.js +26 -2
  23. package/dist/loaders/html-loader.js.map +1 -1
  24. package/dist/loaders/index.d.ts.map +1 -1
  25. package/dist/loaders/index.js.map +1 -1
  26. package/dist/loaders/json-loader.d.ts +1 -0
  27. package/dist/loaders/json-loader.d.ts.map +1 -1
  28. package/dist/loaders/json-loader.js +15 -3
  29. package/dist/loaders/json-loader.js.map +1 -1
  30. package/dist/loaders/markdown-loader.d.ts.map +1 -1
  31. package/dist/loaders/markdown-loader.js +8 -4
  32. package/dist/loaders/markdown-loader.js.map +1 -1
  33. package/dist/loaders/pdf-loader.d.ts +1 -0
  34. package/dist/loaders/pdf-loader.d.ts.map +1 -1
  35. package/dist/loaders/pdf-loader.js +22 -3
  36. package/dist/loaders/pdf-loader.js.map +1 -1
  37. package/dist/loaders/text-loader.js +1 -1
  38. package/dist/loaders/text-loader.js.map +1 -1
  39. package/dist/loaders/web-loader.d.ts +33 -0
  40. package/dist/loaders/web-loader.d.ts.map +1 -1
  41. package/dist/loaders/web-loader.js +262 -8
  42. package/dist/loaders/web-loader.js.map +1 -1
  43. package/dist/rag-builder.d.ts +10 -1
  44. package/dist/rag-builder.d.ts.map +1 -1
  45. package/dist/rag-builder.js +71 -6
  46. package/dist/rag-builder.js.map +1 -1
  47. package/dist/rag-pipeline.d.ts +2 -1
  48. package/dist/rag-pipeline.d.ts.map +1 -1
  49. package/dist/rag-pipeline.js +58 -23
  50. package/dist/rag-pipeline.js.map +1 -1
  51. package/dist/rerankers/cohere-reranker.d.ts +2 -2
  52. package/dist/rerankers/cohere-reranker.d.ts.map +1 -1
  53. package/dist/rerankers/cohere-reranker.js +5 -0
  54. package/dist/rerankers/cohere-reranker.js.map +1 -1
  55. package/dist/rerankers/index.d.ts.map +1 -1
  56. package/dist/rerankers/index.js.map +1 -1
  57. package/dist/rerankers/llm-reranker.d.ts +1 -1
  58. package/dist/rerankers/llm-reranker.d.ts.map +1 -1
  59. package/dist/rerankers/llm-reranker.js +26 -13
  60. package/dist/rerankers/llm-reranker.js.map +1 -1
  61. package/dist/retrievers/chunk-indexer.d.ts +19 -0
  62. package/dist/retrievers/chunk-indexer.d.ts.map +1 -0
  63. package/dist/retrievers/chunk-indexer.js +8 -0
  64. package/dist/retrievers/chunk-indexer.js.map +1 -0
  65. package/dist/retrievers/create-retriever.d.ts +2 -0
  66. package/dist/retrievers/create-retriever.d.ts.map +1 -1
  67. package/dist/retrievers/create-retriever.js +6 -0
  68. package/dist/retrievers/create-retriever.js.map +1 -1
  69. package/dist/retrievers/hybrid-retriever.d.ts +8 -1
  70. package/dist/retrievers/hybrid-retriever.d.ts.map +1 -1
  71. package/dist/retrievers/hybrid-retriever.js +18 -5
  72. package/dist/retrievers/hybrid-retriever.js.map +1 -1
  73. package/dist/retrievers/index.d.ts +2 -0
  74. package/dist/retrievers/index.d.ts.map +1 -1
  75. package/dist/retrievers/index.js +1 -0
  76. package/dist/retrievers/index.js.map +1 -1
  77. package/dist/retrievers/mmr-retriever.d.ts +5 -0
  78. package/dist/retrievers/mmr-retriever.d.ts.map +1 -1
  79. package/dist/retrievers/mmr-retriever.js +34 -8
  80. package/dist/retrievers/mmr-retriever.js.map +1 -1
  81. package/dist/retrievers/multi-query-retriever.d.ts +14 -1
  82. package/dist/retrievers/multi-query-retriever.d.ts.map +1 -1
  83. package/dist/retrievers/multi-query-retriever.js +50 -6
  84. package/dist/retrievers/multi-query-retriever.js.map +1 -1
  85. package/dist/retrievers/similarity-retriever.d.ts.map +1 -1
  86. package/dist/retrievers/similarity-retriever.js +2 -1
  87. package/dist/retrievers/similarity-retriever.js.map +1 -1
  88. package/dist/schema.d.ts.map +1 -1
  89. package/dist/schema.js +16 -2
  90. package/dist/schema.js.map +1 -1
  91. package/dist/tools.d.ts +11 -2
  92. package/dist/tools.d.ts.map +1 -1
  93. package/dist/tools.js +43 -9
  94. package/dist/tools.js.map +1 -1
  95. package/package.json +5 -5
@@ -1 +1 @@
1
- {"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,UAA6B;QACvD,MAAM,OAAO,GAAG,UAAU;aACvB,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAC7C,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAClD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,aAAa,CAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QAC/B,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YAC1C,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY,EACZ,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBACD,IAAI,cAAc,KAAK,CAAC,QAAQ;oBAAE,cAAc,GAAG,CAAC,CAAC;gBAErD,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}
@@ -1,13 +1,26 @@
1
1
  import type { Retriever, RetrievalConfig, RetrievalResult } from '@cogitator-ai/types';
2
+ import { type ChunkIndexer, type IndexedChunk } from './chunk-indexer.js';
2
3
  export interface MultiQueryRetrieverConfig {
3
4
  baseRetriever: Retriever;
4
5
  expandQuery: (query: string) => Promise<string[]>;
6
+ defaultTopK?: number;
7
+ /** Maximum number of generated query variants used in addition to the original query. */
8
+ defaultMaxQueries?: number;
5
9
  }
6
- export declare class MultiQueryRetriever implements Retriever {
10
+ /**
11
+ * Expands the query into variants, retrieves for each in parallel and merges results
12
+ * (best score per chunk). The original query is always searched. If expansion fails the
13
+ * original query alone is used; if every retrieval fails the first error is thrown.
14
+ */
15
+ export declare class MultiQueryRetriever implements Retriever, ChunkIndexer {
7
16
  private readonly baseRetriever;
8
17
  private readonly expandQuery;
18
+ private readonly defaultTopK;
19
+ private readonly defaultMaxQueries?;
9
20
  constructor(config: MultiQueryRetrieverConfig);
21
+ indexChunk(chunk: IndexedChunk): void;
10
22
  retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
23
+ private buildVariants;
11
24
  private deduplicateAndMerge;
12
25
  }
13
26
  //# sourceMappingURL=multi-query-retriever.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnD;AAID,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;gBAEvD,MAAM,EAAE,yBAAyB;IAKvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAe7F,OAAO,CAAC,mBAAmB;CAY5B"}
1
+ {"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE1F,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAID;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,SAAS,EAAE,YAAY;IACjE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;gBAEhC,MAAM,EAAE,yBAAyB;IAO7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAM/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YA0B/E,aAAa;IAwB3B,OAAO,CAAC,mBAAmB;CAY5B"}
@@ -1,20 +1,64 @@
1
+ import { isChunkIndexer } from './chunk-indexer.js';
1
2
  const DEFAULT_TOP_K = 10;
3
+ /**
4
+ * Expands the query into variants, retrieves for each in parallel and merges results
5
+ * (best score per chunk). The original query is always searched. If expansion fails the
6
+ * original query alone is used; if every retrieval fails the first error is thrown.
7
+ */
2
8
  export class MultiQueryRetriever {
3
9
  baseRetriever;
4
10
  expandQuery;
11
+ defaultTopK;
12
+ defaultMaxQueries;
5
13
  constructor(config) {
6
14
  this.baseRetriever = config.baseRetriever;
7
15
  this.expandQuery = config.expandQuery;
16
+ this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
17
+ this.defaultMaxQueries = config.defaultMaxQueries;
18
+ }
19
+ indexChunk(chunk) {
20
+ if (isChunkIndexer(this.baseRetriever)) {
21
+ this.baseRetriever.indexChunk(chunk);
22
+ }
8
23
  }
9
24
  async retrieve(query, options) {
10
- const variants = await this.expandQuery(query);
11
- if (variants.length === 0)
12
- return [];
13
- const allResults = await Promise.all(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
14
- const merged = this.deduplicateAndMerge(allResults.flat());
15
- const topK = options?.topK ?? DEFAULT_TOP_K;
25
+ const variants = await this.buildVariants(query, options?.multiQueryCount);
26
+ const settled = await Promise.allSettled(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
27
+ const fulfilled = settled.filter((outcome) => outcome.status === 'fulfilled');
28
+ if (fulfilled.length === 0) {
29
+ const firstFailure = settled.find((outcome) => outcome.status === 'rejected');
30
+ throw firstFailure?.reason instanceof Error
31
+ ? firstFailure.reason
32
+ : new Error(`MultiQueryRetriever: all retrievals failed: ${String(firstFailure?.reason)}`);
33
+ }
34
+ const merged = this.deduplicateAndMerge(fulfilled.flatMap((outcome) => outcome.value));
35
+ const topK = options?.topK ?? this.defaultTopK;
16
36
  return merged.slice(0, topK);
17
37
  }
38
+ async buildVariants(query, requestedMax) {
39
+ let expanded;
40
+ try {
41
+ expanded = await this.expandQuery(query);
42
+ }
43
+ catch {
44
+ expanded = [];
45
+ }
46
+ const normalized = query.trim().toLowerCase();
47
+ const seen = new Set([normalized]);
48
+ const extra = [];
49
+ for (const variant of expanded) {
50
+ if (typeof variant !== 'string')
51
+ continue;
52
+ const trimmed = variant.trim();
53
+ const key = trimmed.toLowerCase();
54
+ if (!trimmed || seen.has(key))
55
+ continue;
56
+ seen.add(key);
57
+ extra.push(trimmed);
58
+ }
59
+ const max = requestedMax ?? this.defaultMaxQueries;
60
+ return [query, ...(max === undefined ? extra : extra.slice(0, max))];
61
+ }
18
62
  deduplicateAndMerge(results) {
19
63
  const best = new Map();
20
64
  for (const result of results) {
@@ -1 +1 @@
1
- {"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAOA,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAEnE,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,aAAa,CAAC;QAE5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
1
+ {"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAwC,MAAM,oBAAoB,CAAC;AAU1F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IACpB,iBAAiB,CAAU;IAE5C,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpD,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QAE3E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,OAAO,EAAwD,EAAE,CAChE,OAAO,CAAC,MAAM,KAAK,WAAW,CACjC,CAAC;QACF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAC/B,CAAC,OAAO,EAAoC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAC7E,CAAC;YACF,MAAM,YAAY,EAAE,MAAM,YAAY,KAAK;gBACzC,CAAC,CAAC,YAAY,CAAC,MAAM;gBACrB,CAAC,CAAC,IAAI,KAAK,CAAC,+CAA+C,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,YAAqB;QAC9D,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,SAAS;YAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,GAAG,GAAG,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC;QACnD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}
1
+ {"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}
@@ -1,3 +1,4 @@
1
+ import { resultSource } from './chunk-indexer.js';
1
2
  const DEFAULT_TOP_K = 10;
2
3
  const DEFAULT_THRESHOLD = 0.0;
3
4
  export class SimilarityRetriever {
@@ -32,7 +33,7 @@ export class SimilarityRetriever {
32
33
  documentId,
33
34
  content: entry.content,
34
35
  score: entry.score,
35
- source: entry.sourceType,
36
+ source: resultSource(entry.metadata, entry.sourceType),
36
37
  metadata: entry.metadata,
37
38
  };
38
39
  }
@@ -1 +1 @@
1
- {"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
1
+ {"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
@@ -1 +1 @@
1
- {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;iBAK/B,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;EAAyD,CAAC;AAE9F,eAAO,MAAM,qBAAqB;;;;;;;;;;;iBAMhC,CAAC;AAIH,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AACvE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;EAA6C,CAAC;AAEjF,eAAO,MAAM,oBAAoB;;;;;;;;;iBAU7B,CAAC;AAEL,eAAO,MAAM,uBAAuB;;;;;EAAyD,CAAC;AAE9F,eAAO,MAAM,qBAAqB;;;;;;;;;;;iBAe9B,CAAC;AAIL,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIlC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AACvE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC"}
package/dist/schema.js CHANGED
@@ -1,18 +1,32 @@
1
1
  import { z } from 'zod';
2
2
  export const ChunkingStrategySchema = z.enum(['fixed', 'recursive', 'semantic']);
3
- export const ChunkingConfigSchema = z.object({
3
+ export const ChunkingConfigSchema = z
4
+ .object({
4
5
  strategy: ChunkingStrategySchema,
5
6
  chunkSize: z.number().int().positive(),
6
7
  chunkOverlap: z.number().int().nonnegative().default(0),
7
8
  separators: z.array(z.string()).optional(),
9
+ })
10
+ .refine((d) => d.chunkOverlap < d.chunkSize, {
11
+ message: 'chunkOverlap must be less than chunkSize',
12
+ path: ['chunkOverlap'],
8
13
  });
9
14
  export const RetrievalStrategySchema = z.enum(['similarity', 'mmr', 'hybrid', 'multi-query']);
10
- export const RetrievalConfigSchema = z.object({
15
+ export const RetrievalConfigSchema = z
16
+ .object({
11
17
  strategy: RetrievalStrategySchema.default('similarity'),
12
18
  topK: z.number().int().positive().default(10),
13
19
  threshold: z.number().min(0).max(1).default(0.0),
14
20
  mmrLambda: z.number().min(0).max(1).optional(),
15
21
  multiQueryCount: z.number().int().positive().optional(),
22
+ })
23
+ .refine((d) => d.strategy !== 'mmr' || d.mmrLambda !== undefined, {
24
+ message: 'mmrLambda is required when strategy is "mmr"',
25
+ path: ['mmrLambda'],
26
+ })
27
+ .refine((d) => d.strategy !== 'multi-query' || d.multiQueryCount !== undefined, {
28
+ message: 'multiQueryCount is required when strategy is "multi-query"',
29
+ path: ['multiQueryCount'],
16
30
  });
17
31
  const retrievalDefaults = RetrievalConfigSchema.parse({});
18
32
  export const RerankingConfigSchema = z.object({
@@ -1 +1 @@
1
- {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,QAAQ,EAAE,sBAAsB;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,QAAQ,EAAE,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC;IACvD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAC3D,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC"}
1
+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,QAAQ,EAAE,sBAAsB;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACtC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,SAAS,EAAE;IAC3C,OAAO,EAAE,0CAA0C;IACnD,IAAI,EAAE,CAAC,cAAc,CAAC;CACvB,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,QAAQ,EAAE,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC;IACvD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,EAAE;IAChE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,WAAW,CAAC;CACpB,CAAC;KACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS,EAAE;IAC9E,OAAO,EAAE,4DAA4D;IACrE,IAAI,EAAE,CAAC,iBAAiB,CAAC;CAC1B,CAAC,CAAC;AAEL,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC,iBAAiB,CAAC;IAC3D,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC"}
package/dist/tools.d.ts CHANGED
@@ -17,7 +17,16 @@ declare const IngestParamsSchema: z.ZodObject<{
17
17
  }, z.core.$strip>;
18
18
  type IngestParams = z.infer<typeof IngestParamsSchema>;
19
19
  export declare function createSearchTool(pipeline: RAGPipeline): RAGTool<SearchParams>;
20
- export declare function createIngestTool(pipeline: RAGPipeline): RAGTool<IngestParams>;
21
- export declare function ragTools(pipeline: RAGPipeline): [RAGTool<SearchParams>, RAGTool<IngestParams>];
20
+ export interface IngestToolOptions {
21
+ /**
22
+ * Restrict local sources to these directories (symlinks are resolved). When set, any
23
+ * path outside them is rejected. Strongly recommended when the tool is exposed to an LLM.
24
+ */
25
+ allowedRoots?: string[];
26
+ /** Allow http(s) URLs as sources. Defaults to `true`. */
27
+ allowUrls?: boolean;
28
+ }
29
+ export declare function createIngestTool(pipeline: RAGPipeline, options?: IngestToolOptions): RAGTool<IngestParams>;
30
+ export declare function ragTools(pipeline: RAGPipeline, ingestOptions?: IngestToolOptions): [RAGTool<SearchParams>, RAGTool<IngestParams>];
22
31
  export {};
23
32
  //# sourceMappingURL=tools.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAc7E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAc7E;AAED,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,WAAW,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAE9F"}
1
+ {"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,MAAM,WAAW,OAAO,CAAC,OAAO,GAAG,OAAO;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAED,QAAA,MAAM,kBAAkB;;;;iBAItB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,QAAA,MAAM,kBAAkB;;iBAEtB,CAAC;AAEH,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEvD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAe7E;AAED,MAAM,WAAW,iBAAiB;IAChC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,yDAAyD;IACzD,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAiCD,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,YAAY,CAAC,CAgBvB;AAED,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,WAAW,EACrB,aAAa,CAAC,EAAE,iBAAiB,GAChC,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAEhD"}
package/dist/tools.js CHANGED
@@ -1,19 +1,22 @@
1
+ import { realpath } from 'node:fs/promises';
2
+ import { isAbsolute, relative, resolve } from 'node:path';
1
3
  import { z } from 'zod';
2
4
  const SearchParamsSchema = z.object({
3
- query: z.string(),
4
- limit: z.number().int().positive().optional(),
5
- threshold: z.number().min(0).max(1).optional(),
5
+ query: z.string().min(1).describe('Search query for the knowledge base'),
6
+ limit: z.number().int().positive().optional().describe('Maximum number of results to return'),
7
+ threshold: z.number().min(0).max(1).optional().describe('Minimum relevance score (0-1)'),
6
8
  });
7
9
  const IngestParamsSchema = z.object({
8
- source: z.string(),
10
+ source: z.string().min(1).describe('File path, directory or URL to ingest'),
9
11
  });
10
12
  export function createSearchTool(pipeline) {
11
13
  return {
12
14
  name: 'rag_search',
13
15
  description: 'Search the knowledge base using semantic search',
14
16
  parameters: SearchParamsSchema,
15
- execute: async ({ query, limit, threshold }) => {
17
+ execute: async (params) => {
16
18
  try {
19
+ const { query, limit, threshold } = SearchParamsSchema.parse(params);
17
20
  const results = await pipeline.query(query, { topK: limit, threshold });
18
21
  return { success: true, query, results, count: results.length };
19
22
  }
@@ -23,13 +26,44 @@ export function createSearchTool(pipeline) {
23
26
  },
24
27
  };
25
28
  }
26
- export function createIngestTool(pipeline) {
29
+ function isUrl(source) {
30
+ return /^[a-z][a-z0-9+.-]*:\/\//i.test(source);
31
+ }
32
+ async function canonicalPath(path) {
33
+ try {
34
+ return await realpath(path);
35
+ }
36
+ catch {
37
+ return resolve(path);
38
+ }
39
+ }
40
+ async function assertAllowedSource(source, options) {
41
+ if (isUrl(source)) {
42
+ if (options.allowUrls === false) {
43
+ throw new Error('URL sources are not allowed');
44
+ }
45
+ return;
46
+ }
47
+ if (!options.allowedRoots)
48
+ return;
49
+ const target = await canonicalPath(source);
50
+ for (const root of options.allowedRoots) {
51
+ const canonicalRoot = await canonicalPath(root);
52
+ const rel = relative(canonicalRoot, target);
53
+ if (rel === '' || (!rel.startsWith('..') && !isAbsolute(rel)))
54
+ return;
55
+ }
56
+ throw new Error(`Source "${source}" is outside the allowed directories`);
57
+ }
58
+ export function createIngestTool(pipeline, options = {}) {
27
59
  return {
28
60
  name: 'rag_ingest',
29
61
  description: 'Ingest documents into the knowledge base',
30
62
  parameters: IngestParamsSchema,
31
- execute: async ({ source }) => {
63
+ execute: async (params) => {
32
64
  try {
65
+ const { source } = IngestParamsSchema.parse(params);
66
+ await assertAllowedSource(source, options);
33
67
  const { documents, chunks } = await pipeline.ingest(source);
34
68
  return { success: true, source, documents, chunks };
35
69
  }
@@ -39,7 +73,7 @@ export function createIngestTool(pipeline) {
39
73
  },
40
74
  };
41
75
  }
42
- export function ragTools(pipeline) {
43
- return [createSearchTool(pipeline), createIngestTool(pipeline)];
76
+ export function ragTools(pipeline, ingestOptions) {
77
+ return [createSearchTool(pipeline), createIngestTool(pipeline, ingestOptions)];
44
78
  }
45
79
  //# sourceMappingURL=tools.js.map
package/dist/tools.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC7C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC/C,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;CACnB,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7C,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,QAAqB;IAC5C,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAClE,CAAC"}
1
+ {"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC7F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACzF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;CAC5E,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAYD,SAAS,KAAK,CAAC,MAAc;IAC3B,OAAO,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAA0B;IAC3E,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO;IAElC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;IACxE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,sCAAsC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAqB,EACrB,UAA6B,EAAE;IAE/B,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,QAAqB,EACrB,aAAiC;IAEjC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACjF,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cogitator-ai/rag",
3
- "version": "0.1.9",
3
+ "version": "0.2.0",
4
4
  "description": "Retrieval Augmented Generation for Cogitator AI agents",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -17,10 +17,10 @@
17
17
  "dependencies": {
18
18
  "nanoid": "^5.0.4",
19
19
  "zod": "^4.3.6",
20
- "@cogitator-ai/types": "0.22.2"
20
+ "@cogitator-ai/types": "0.23.0"
21
21
  },
22
22
  "peerDependencies": {
23
- "@cogitator-ai/memory": "0.6.21"
23
+ "@cogitator-ai/memory": ">=0.7.0"
24
24
  },
25
25
  "peerDependenciesMeta": {
26
26
  "@cogitator-ai/memory": {
@@ -37,7 +37,7 @@
37
37
  "@types/pdf-parse": "^1.1.5",
38
38
  "typescript": "^5.7.2",
39
39
  "vitest": "^4.0.18",
40
- "@cogitator-ai/memory": "0.6.21"
40
+ "@cogitator-ai/memory": "0.7.0"
41
41
  },
42
42
  "repository": {
43
43
  "type": "git",
@@ -50,7 +50,7 @@
50
50
  },
51
51
  "license": "MIT",
52
52
  "scripts": {
53
- "build": "tsc",
53
+ "build": "tsc && tsx ../../scripts/fix-esm-imports.ts .",
54
54
  "dev": "tsc --watch",
55
55
  "clean": "rm -rf dist",
56
56
  "typecheck": "tsc --noEmit",