@hazeljs/rag 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/errors/rag.error.test.d.ts +2 -0
- package/dist/__tests__/errors/rag.error.test.d.ts.map +1 -0
- package/dist/__tests__/errors/rag.error.test.js +95 -0
- package/dist/__tests__/errors/rag.error.test.js.map +1 -0
- package/dist/__tests__/utils/debug.test.d.ts +2 -0
- package/dist/__tests__/utils/debug.test.d.ts.map +1 -0
- package/dist/__tests__/utils/debug.test.js +99 -0
- package/dist/__tests__/utils/debug.test.js.map +1 -0
- package/dist/errors/rag.error.d.ts +33 -0
- package/dist/errors/rag.error.d.ts.map +1 -0
- package/dist/errors/rag.error.js +64 -0
- package/dist/errors/rag.error.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -1
- package/dist/rag-pipeline.d.ts +39 -3
- package/dist/rag-pipeline.d.ts.map +1 -1
- package/dist/rag-pipeline.js +127 -7
- package/dist/rag-pipeline.js.map +1 -1
- package/dist/rag.service.d.ts +60 -11
- package/dist/rag.service.d.ts.map +1 -1
- package/dist/rag.service.js +390 -28
- package/dist/rag.service.js.map +1 -1
- package/dist/utils/debug.d.ts +29 -0
- package/dist/utils/debug.d.ts.map +1 -0
- package/dist/utils/debug.js +82 -0
- package/dist/utils/debug.js.map +1 -0
- package/package.json +4 -4
package/dist/rag-pipeline.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,mCAOiB;
|
|
1
|
+
{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,mCAOiB;AACjB,2CAAsD;AACtD,sEAAkE;AAClE,sEAAkE;AAClE,6EAAwE;AACxE,sFAAiF;AA4BjF,MAAa,WAAW;IAMtB,YAAY,MAAiB,EAAE,WAAyB;QAFhD,gBAAW,GAAY,KAAK,CAAC;QAGnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,WAAI,EAAE,CAAC;IACzB,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,IAAI,CAAC,QAAgC,EAAE;QAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAE5C,6BAA6B;QAC7B,IAAI,iBAAiB,CAAC;QACtB,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;gBAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CACb,kDAAkD;wBAChD,+DAA+D,CAClE,CAAC;gBACJ,CAAC;gBACD,iBAAiB,GAAG,IAAI,oCAAgB,CAAC;oBACvC,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,cAAc;iBAC5B,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YACD,KAAK,QAAQ,CAAC;YACd,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;gBAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CACb,kDAAkD;wBAChD,+DAA+D,CAClE,CAAC;gBACJ,CAAC;gBACD,iBAAiB,GAAG,IAAI,oCAAgB,CAAC;oBACvC,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,cAAc;iBAC5B,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;QACH,CAAC;QAED,iEAAiE;QACjE,MAAM,WAAW,GAAG,IAAI,uCAAiB,CAAC,iBAAiB,CAAC,CAAC;QAE7D,wBAAwB;QACxB,MAAM,YAAY,GAAG,IAAI,+CAAqB,CAAC;YAC7C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YAClC,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,GAAG;SACxC,CAAC,CAAC;QAEH,OAAO,IAAI,WAAW,CACpB;YACE,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;SACtB,EACD,KAAK,CAAC,GAAG,CACV,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,IAAI,aAAa,GAAG,SAAS,CAAC;QAE9B,iDAAiD;QACjD,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;QAEtE,kCAAkC;QAClC,MAAM,QAAQ,GAAmB,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9D,EAAE,EAAE,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,MAAM,EAAE,GAAG,CAAC,OAAO;iBAChB,WAAW,EAAE;iBACb,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;iBACxB,KAAK,CAAC,KAAK,CAAC;iBACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC/B,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAExB,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,KAAa,EAAE,UAA2B,EAAE;QACtD,MAAM,EACJ,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,EAC5B,QAAQ,GAAG,yBAAiB,CAAC,UAAU,EACvC,SAAS,EACT,cAAc,GAAG,IAAI,EACrB,GAAG,aAAa,EACjB,GAAG,OAAO,CAAC;QAEZ,8BAA8B;QAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,GAAG,aAAa,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;QAEjF,6BAA6B;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3C,yCAAyC;QACzC,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,WAAW,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,OAAO,CAAC,CAAC,2BAA2B;QAC/C,CAAC;QAED,OAAO;YACL,MAAM;YACN,OAAO;YACP,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SACvC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CACZ,KAAa,EACb,UAA2B,EAAE,EAC7B,WAA8B,yBAAiB,CAAC,UAAU;QAE1D,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,yBAAiB,CAAC,UAAU;gBAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAExD,KAAK,yBAAiB,CAAC,GAAG;gBACxB,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAE9C,KAAK,yBAAiB,CAAC,MAAM;gBAC3B,OAAO,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAE7C;gBACE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAC3B,KAAa,EACb,UAA2B,EAAE;QAE7B,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,aAAa,EAAE,GAAG,OAAO,CAAC;QAE/C,iDAAiD;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE;YAC7D,GAAG,aAAa;YAChB,IAAI,EAAE,IAAI,GAAG,CAAC;YACd,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,2EAA2E;QAC3E,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,gBAAgB;QAChB,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,0CAA0C;QAE9D,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAC1B,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAChC,IAAI,CAAC,SAAS,CAAC,SAAS;oBAAE,SAAS;gBAEnC,+BAA+B;gBAC/B,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC;gBAElC,yDAAyD;gBACzD,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,KAAK,MAAM,WAAW,IAAI,QAAQ,EAAE,CAAC;oBACnC,IAAI,CAAC,WAAW,CAAC,SAAS;wBAAE,SAAS;oBACrC,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,SAAS,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;oBACrF,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;gBACtD,CAAC;gBAED,YAAY;gBACZ,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,aAAa,CAAC;gBAEnE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,CAAC,CAAC;gBACd,CAAC;YACH,CAAC;YAED,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;gBACnC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,cAAc,CAC1B,KAAa,EACb,UAA2B,EAAE;QAE7B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;QAEnD,2BAA2B;QAC3B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE;YAChE,GAAG,OAAO;YACV,IAAI,EAAE,IAAI,GAAG,CAAC;SACf,CAAC,CAAC;QAEH,4DAA4D;QAC5D,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;QAEzD,6BAA6B;QAC7B,MAAM,SAAS,GAAG,CAAC,KAA2C,EAAuB,EAAE;YACrF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,GAAG,EAAkB,CAAC;YACzD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC;YAC7B,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QACzF,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QAEhD,MAAM,YAAY,GAAG,GAAG,CAAC;QACzB,MAAM,aAAa,GAAG,GAAG,CAAC;QAE1B,cAAc;QACd,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,IAAI,EAAE,EAAE,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAyC,EAAE,CAAC;QAEvD,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,GAAG,EAAE,GAAG,aAAa,GAAG,EAAE,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAExC,wCAAwC;QACxC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,KAAK;aACT,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;aACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrC,IAAI,QAAQ;gBAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;YACrD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,OAAuB;QAC1C,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;YACnB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,GAAG,GAAG,CAAC,KAAK,MAAM,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;QACrD,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,OAAe,EAAE,MAAc;QACzE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAElF,OAAO,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,CAAW,EAAE,CAAW;QAC/C,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,GAAa;QACjC,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;CACF;AAtWD,kCAsWC"}
|
package/dist/rag.service.d.ts
CHANGED
|
@@ -11,10 +11,21 @@ export interface RAGServiceConfig {
|
|
|
11
11
|
textSplitter?: TextSplitter;
|
|
12
12
|
llmFunction?: LLMFunction;
|
|
13
13
|
topK?: number;
|
|
14
|
+
/** Custom query generator for multi-query retrieval (uses simple variations if not set). */
|
|
15
|
+
queryGenerator?: (question: string, numQueries: number) => Promise<string[]>;
|
|
16
|
+
/** Weights for hybrid search. Defaults: vectorWeight=0.7, keywordWeight=0.3. */
|
|
17
|
+
hybridWeights?: {
|
|
18
|
+
vectorWeight?: number;
|
|
19
|
+
keywordWeight?: number;
|
|
20
|
+
};
|
|
14
21
|
}
|
|
15
22
|
export declare class RAGService {
|
|
16
23
|
private config;
|
|
17
24
|
private pipeline;
|
|
25
|
+
private multiQueryRetrieval?;
|
|
26
|
+
private hybridRetrieval?;
|
|
27
|
+
private bm25;
|
|
28
|
+
private conversationHistory;
|
|
18
29
|
constructor(config: RAGServiceConfig);
|
|
19
30
|
/**
|
|
20
31
|
* Initialize the RAG service
|
|
@@ -24,6 +35,27 @@ export declare class RAGService {
|
|
|
24
35
|
* Index a document or multiple documents
|
|
25
36
|
*/
|
|
26
37
|
index(documents: Document | Document[]): Promise<string[]>;
|
|
38
|
+
/**
|
|
39
|
+
* Ingest documents from a file path, URL, or directory.
|
|
40
|
+
* Auto-detects the source type and uses the appropriate loader.
|
|
41
|
+
*
|
|
42
|
+
* @example
|
|
43
|
+
* ```ts
|
|
44
|
+
* await rag.ingest('./docs/guide.pdf');
|
|
45
|
+
* await rag.ingest('./data/faq.csv');
|
|
46
|
+
* await rag.ingest('https://example.com/page');
|
|
47
|
+
* await rag.ingest('./knowledge-base/'); // loads entire directory
|
|
48
|
+
* ```
|
|
49
|
+
*/
|
|
50
|
+
ingest(source: string): Promise<string[]>;
|
|
51
|
+
/**
|
|
52
|
+
* Auto-detect source type and load documents.
|
|
53
|
+
*/
|
|
54
|
+
private loadSource;
|
|
55
|
+
/**
|
|
56
|
+
* Extract file extension from a path.
|
|
57
|
+
*/
|
|
58
|
+
private getExtension;
|
|
27
59
|
/**
|
|
28
60
|
* Search for similar documents
|
|
29
61
|
*/
|
|
@@ -47,24 +79,39 @@ export declare class RAGService {
|
|
|
47
79
|
}>;
|
|
48
80
|
/**
|
|
49
81
|
* Multi-query RAG
|
|
50
|
-
* Generates multiple search queries and combines
|
|
82
|
+
* Generates multiple search queries from a single question and combines
|
|
83
|
+
* deduplicated results, ranked by frequency and average score.
|
|
84
|
+
*/
|
|
85
|
+
multiQuery(question: string, numQueries?: number): Promise<SearchResult[]>;
|
|
86
|
+
/**
|
|
87
|
+
* Compress retrieved context by removing low-relevance and redundant results.
|
|
88
|
+
* Uses LLM-based compression when available, otherwise applies score-based filtering
|
|
89
|
+
* with deduplication.
|
|
51
90
|
*/
|
|
52
|
-
|
|
91
|
+
compress(documents: SearchResult[], query: string): Promise<SearchResult[]>;
|
|
53
92
|
/**
|
|
54
|
-
*
|
|
93
|
+
* Jaccard similarity between two strings (token-level).
|
|
55
94
|
*/
|
|
56
|
-
|
|
95
|
+
private jaccardSimilarity;
|
|
57
96
|
/**
|
|
58
|
-
* Self-query with automatic metadata extraction
|
|
97
|
+
* Self-query with automatic metadata extraction.
|
|
98
|
+
* Extracts filter conditions from natural language when an LLM is available,
|
|
99
|
+
* then applies them as metadata filters on the vector search.
|
|
59
100
|
*/
|
|
60
101
|
selfQuery(naturalLanguageQuery: string): Promise<SearchResult[]>;
|
|
61
102
|
/**
|
|
62
|
-
* Conversational RAG with session memory
|
|
103
|
+
* Conversational RAG with session memory.
|
|
104
|
+
* Maintains per-session conversation history and rewrites the user's
|
|
105
|
+
* follow-up question into a standalone query before retrieval.
|
|
63
106
|
*/
|
|
64
|
-
chat(message: string,
|
|
107
|
+
chat(message: string, sessionId: string): Promise<{
|
|
65
108
|
answer: string;
|
|
66
109
|
sources: SearchResult[];
|
|
67
110
|
}>;
|
|
111
|
+
/**
|
|
112
|
+
* Clear conversation history for a session.
|
|
113
|
+
*/
|
|
114
|
+
clearChat(sessionId: string): void;
|
|
68
115
|
/**
|
|
69
116
|
* Hybrid search combining vector and keyword search
|
|
70
117
|
*/
|
|
@@ -73,13 +120,15 @@ export declare class RAGService {
|
|
|
73
120
|
keywordWeight?: number;
|
|
74
121
|
}): Promise<SearchResult[]>;
|
|
75
122
|
/**
|
|
76
|
-
* Rerank search results
|
|
123
|
+
* Rerank search results using cross-encoder-style LLM scoring.
|
|
124
|
+
* When no LLM is configured, falls back to BM25-boosted re-scoring.
|
|
77
125
|
*/
|
|
78
|
-
rerank(results: SearchResult[],
|
|
126
|
+
rerank(results: SearchResult[], query: string, topN?: number): Promise<SearchResult[]>;
|
|
79
127
|
/**
|
|
80
|
-
* Ensemble retrieval combining multiple
|
|
128
|
+
* Ensemble retrieval combining multiple retrieval strategies.
|
|
129
|
+
* Runs each strategy, normalizes scores, applies weights, and fuses results.
|
|
81
130
|
*/
|
|
82
|
-
ensemble(query: string,
|
|
131
|
+
ensemble(query: string, methods: RetrievalStrategy[], weights?: number[]): Promise<SearchResult[]>;
|
|
83
132
|
/**
|
|
84
133
|
* Time-weighted retrieval favoring recent documents
|
|
85
134
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rag.service.d.ts","sourceRoot":"","sources":["../src/rag.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE1D,OAAO,6BAA6B,CAAC;AAErC,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,iBAAiB,EAClB,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"rag.service.d.ts","sourceRoot":"","sources":["../src/rag.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAe,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE1D,OAAO,6BAA6B,CAAC;AAErC,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,iBAAiB,EAClB,MAAM,SAAS,CAAC;AAejB,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,WAAW,CAAC;IACzB,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,4FAA4F;IAC5F,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,gFAAgF;IAChF,aAAa,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACnE;AAED,qBACa,UAAU;IAOT,OAAO,CAAC,MAAM;IAN1B,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAClD,OAAO,CAAC,eAAe,CAAC,CAAwB;IAChD,OAAO,CAAC,IAAI,CAAO;IACnB,OAAO,CAAC,mBAAmB,CAAoE;gBAE3E,MAAM,EAAE,gBAAgB;IA0B5C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC;;OAEG;IACG,KAAK,CAAC,SAAS,EAAE,QAAQ,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKhE;;;;;;;;;;;OAWG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAS/C;;OAEG;YACW,UAAU;IAyDxB;;OAEG;IACH,OAAO,CAAC,YAAY;IAOpB;;OAEG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,iBAAiB,CAAA;KAAE,GACxD,OAAO,CAAC,YAAY,EAAE,CAAC;IAQ1B;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAI9E;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAkBhF;;OAEG;IACG,GAAG,CACP,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC;IAQvD;;;;OAIG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAE,MAAU,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAiBnF;;;;OAIG;IACG,QAAQ,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAmDjF;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAQzB;;;;OAIG;IACG,SAAS,CAAC,oBAAoB,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAoCtE;;;;OAIG;IACG,IAAI,CACR,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC;IAqDvD;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIlC;;OAEG;IACG,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,GACzE,OAAO,CAAC,YAAY,EAAE,CAAC;IAI1B;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAiE5F;;;OAGG;IACG,QAAQ,CACZ,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,iBAAiB,EAAE,EAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiD1B;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAmBpF;;OAEG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
package/dist/rag.service.js
CHANGED
|
@@ -3,12 +3,45 @@
|
|
|
3
3
|
* RAG Service
|
|
4
4
|
* Main service for RAG operations in HazelJS
|
|
5
5
|
*/
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
+
}) : function(o, v) {
|
|
20
|
+
o["default"] = v;
|
|
21
|
+
});
|
|
6
22
|
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
7
23
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
8
24
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
9
25
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
10
26
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
11
27
|
};
|
|
28
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
29
|
+
var ownKeys = function(o) {
|
|
30
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
31
|
+
var ar = [];
|
|
32
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
33
|
+
return ar;
|
|
34
|
+
};
|
|
35
|
+
return ownKeys(o);
|
|
36
|
+
};
|
|
37
|
+
return function (mod) {
|
|
38
|
+
if (mod && mod.__esModule) return mod;
|
|
39
|
+
var result = {};
|
|
40
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
41
|
+
__setModuleDefault(result, mod);
|
|
42
|
+
return result;
|
|
43
|
+
};
|
|
44
|
+
})();
|
|
12
45
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
13
46
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
14
47
|
};
|
|
@@ -20,15 +53,39 @@ const prompts_1 = require("@hazeljs/prompts");
|
|
|
20
53
|
require("./prompts/rag-answer.prompt");
|
|
21
54
|
const rag_answer_prompt_1 = require("./prompts/rag-answer.prompt");
|
|
22
55
|
const types_1 = require("./types");
|
|
56
|
+
const multi_query_1 = require("./retrieval/multi-query");
|
|
57
|
+
const hybrid_search_1 = require("./retrieval/hybrid-search");
|
|
58
|
+
const bm25_1 = require("./retrieval/bm25");
|
|
59
|
+
const text_file_loader_1 = require("./loaders/text-file.loader");
|
|
60
|
+
const json_file_loader_1 = require("./loaders/json-file.loader");
|
|
61
|
+
const csv_file_loader_1 = require("./loaders/csv-file.loader");
|
|
62
|
+
const markdown_file_loader_1 = require("./loaders/markdown-file.loader");
|
|
63
|
+
const html_file_loader_1 = require("./loaders/html-file.loader");
|
|
64
|
+
const directory_loader_1 = require("./loaders/directory.loader");
|
|
65
|
+
const web_loader_1 = require("./loaders/web.loader");
|
|
66
|
+
const debug_1 = require("./utils/debug");
|
|
67
|
+
const dbg = (0, debug_1.debug)('rag');
|
|
23
68
|
let RAGService = class RAGService {
|
|
24
69
|
constructor(config) {
|
|
25
70
|
this.config = config;
|
|
71
|
+
this.conversationHistory = new Map();
|
|
26
72
|
this.pipeline = new rag_pipeline_1.RAGPipeline({
|
|
27
73
|
vectorStore: config.vectorStore,
|
|
28
74
|
embeddingProvider: config.embeddingProvider,
|
|
29
75
|
textSplitter: config.textSplitter,
|
|
30
76
|
topK: config.topK,
|
|
31
77
|
}, config.llmFunction);
|
|
78
|
+
// Initialize BM25 for keyword search
|
|
79
|
+
this.bm25 = new bm25_1.BM25();
|
|
80
|
+
// Initialize multi-query retrieval
|
|
81
|
+
this.multiQueryRetrieval = new multi_query_1.MultiQueryRetrieval(config.vectorStore, {
|
|
82
|
+
customGenerator: config.queryGenerator,
|
|
83
|
+
});
|
|
84
|
+
// Initialize hybrid search
|
|
85
|
+
this.hybridRetrieval = new hybrid_search_1.HybridSearchRetrieval(config.vectorStore, {
|
|
86
|
+
vectorWeight: config.hybridWeights?.vectorWeight ?? 0.7,
|
|
87
|
+
keywordWeight: config.hybridWeights?.keywordWeight ?? 0.3,
|
|
88
|
+
});
|
|
32
89
|
}
|
|
33
90
|
/**
|
|
34
91
|
* Initialize the RAG service
|
|
@@ -43,12 +100,99 @@ let RAGService = class RAGService {
|
|
|
43
100
|
const docs = Array.isArray(documents) ? documents : [documents];
|
|
44
101
|
return this.pipeline.addDocuments(docs);
|
|
45
102
|
}
|
|
103
|
+
/**
|
|
104
|
+
* Ingest documents from a file path, URL, or directory.
|
|
105
|
+
* Auto-detects the source type and uses the appropriate loader.
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```ts
|
|
109
|
+
* await rag.ingest('./docs/guide.pdf');
|
|
110
|
+
* await rag.ingest('./data/faq.csv');
|
|
111
|
+
* await rag.ingest('https://example.com/page');
|
|
112
|
+
* await rag.ingest('./knowledge-base/'); // loads entire directory
|
|
113
|
+
* ```
|
|
114
|
+
*/
|
|
115
|
+
async ingest(source) {
|
|
116
|
+
dbg('ingest start source=%s', source);
|
|
117
|
+
const docs = await this.loadSource(source);
|
|
118
|
+
dbg('ingest loaded docs=%d', docs.length);
|
|
119
|
+
const ids = await this.index(docs);
|
|
120
|
+
dbg('ingest complete ids=%d', ids.length);
|
|
121
|
+
return ids;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Auto-detect source type and load documents.
|
|
125
|
+
*/
|
|
126
|
+
async loadSource(source) {
|
|
127
|
+
// URL detection
|
|
128
|
+
if (source.startsWith('http://') || source.startsWith('https://')) {
|
|
129
|
+
const loader = new web_loader_1.WebLoader({ url: source });
|
|
130
|
+
return loader.load();
|
|
131
|
+
}
|
|
132
|
+
// Detect if source is a directory (ends with / or has no extension)
|
|
133
|
+
const ext = this.getExtension(source);
|
|
134
|
+
if (!ext || source.endsWith('/')) {
|
|
135
|
+
const loader = new directory_loader_1.DirectoryLoader({ path: source });
|
|
136
|
+
return loader.load();
|
|
137
|
+
}
|
|
138
|
+
// File-based loading by extension
|
|
139
|
+
switch (ext) {
|
|
140
|
+
case '.txt':
|
|
141
|
+
case '.log':
|
|
142
|
+
return new text_file_loader_1.TextFileLoader({ path: source }).load();
|
|
143
|
+
case '.json':
|
|
144
|
+
return new json_file_loader_1.JSONFileLoader({ path: source }).load();
|
|
145
|
+
case '.csv':
|
|
146
|
+
return new csv_file_loader_1.CSVFileLoader({ path: source }).load();
|
|
147
|
+
case '.md':
|
|
148
|
+
case '.mdx':
|
|
149
|
+
return new markdown_file_loader_1.MarkdownFileLoader({ path: source }).load();
|
|
150
|
+
case '.html':
|
|
151
|
+
case '.htm':
|
|
152
|
+
return new html_file_loader_1.HTMLFileLoader({ path: source }).load();
|
|
153
|
+
case '.pdf': {
|
|
154
|
+
// Optional dependency — dynamic import
|
|
155
|
+
try {
|
|
156
|
+
const { PdfLoader } = await Promise.resolve().then(() => __importStar(require('./loaders/pdf.loader')));
|
|
157
|
+
return new PdfLoader({ path: source }).load();
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
throw new Error(`PDF loading requires the 'pdf-parse' package. Install it with: npm install pdf-parse`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
case '.docx': {
|
|
164
|
+
try {
|
|
165
|
+
const { DocxLoader } = await Promise.resolve().then(() => __importStar(require('./loaders/docx.loader')));
|
|
166
|
+
return new DocxLoader({ path: source }).load();
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
throw new Error(`DOCX loading requires the 'mammoth' package. Install it with: npm install mammoth`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
default:
|
|
173
|
+
// Fallback: treat as text file
|
|
174
|
+
return new text_file_loader_1.TextFileLoader({ path: source }).load();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Extract file extension from a path.
|
|
179
|
+
*/
|
|
180
|
+
getExtension(filePath) {
|
|
181
|
+
const dotIndex = filePath.lastIndexOf('.');
|
|
182
|
+
const slashIndex = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf('\\'));
|
|
183
|
+
if (dotIndex <= slashIndex || dotIndex === -1)
|
|
184
|
+
return '';
|
|
185
|
+
return filePath.slice(dotIndex).toLowerCase();
|
|
186
|
+
}
|
|
46
187
|
/**
|
|
47
188
|
* Search for similar documents
|
|
48
189
|
*/
|
|
49
190
|
async search(query, options) {
|
|
191
|
+
dbg('search query=%s strategy=%s', query, options?.strategy || 'similarity');
|
|
50
192
|
const { strategy, ...queryOptions } = options || {};
|
|
51
|
-
|
|
193
|
+
const results = await this.pipeline.retrieve(query, queryOptions, strategy);
|
|
194
|
+
dbg('search results=%d', results.length);
|
|
195
|
+
return results;
|
|
52
196
|
}
|
|
53
197
|
/**
|
|
54
198
|
* Retrieve relevant context for a query
|
|
@@ -76,41 +220,169 @@ let RAGService = class RAGService {
|
|
|
76
220
|
* Full RAG pipeline: retrieve + generate
|
|
77
221
|
*/
|
|
78
222
|
async ask(query, options) {
|
|
223
|
+
dbg('ask query=%s', query);
|
|
79
224
|
const sources = await this.retrieve(query, options);
|
|
80
225
|
const answer = await this.generate(query, sources);
|
|
226
|
+
dbg('ask complete answer_len=%d sources=%d', answer.length, sources.length);
|
|
81
227
|
return { answer, sources };
|
|
82
228
|
}
|
|
83
229
|
/**
|
|
84
230
|
* Multi-query RAG
|
|
85
|
-
* Generates multiple search queries and combines
|
|
231
|
+
* Generates multiple search queries from a single question and combines
|
|
232
|
+
* deduplicated results, ranked by frequency and average score.
|
|
86
233
|
*/
|
|
87
|
-
async multiQuery(question,
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
234
|
+
async multiQuery(question, numQueries = 3) {
|
|
235
|
+
dbg('multiQuery question=%s numQueries=%d', question, numQueries);
|
|
236
|
+
if (!this.multiQueryRetrieval) {
|
|
237
|
+
return this.search(question, { topK: 10 });
|
|
238
|
+
}
|
|
239
|
+
// Override numQueries on the fly
|
|
240
|
+
const retrieval = new multi_query_1.MultiQueryRetrieval(this.config.vectorStore, {
|
|
241
|
+
numQueries,
|
|
242
|
+
customGenerator: this.config.queryGenerator,
|
|
243
|
+
});
|
|
244
|
+
const results = await retrieval.retrieve(question, { topK: numQueries * 5 });
|
|
245
|
+
dbg('multiQuery results=%d', results.length);
|
|
246
|
+
return results;
|
|
91
247
|
}
|
|
92
248
|
/**
|
|
93
|
-
* Compress retrieved context
|
|
249
|
+
* Compress retrieved context by removing low-relevance and redundant results.
|
|
250
|
+
* Uses LLM-based compression when available, otherwise applies score-based filtering
|
|
251
|
+
* with deduplication.
|
|
94
252
|
*/
|
|
95
|
-
async compress(documents,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
253
|
+
async compress(documents, query) {
|
|
254
|
+
if (documents.length === 0)
|
|
255
|
+
return [];
|
|
256
|
+
// Step 1: Remove results below a relevance threshold (adaptive: median * 0.6)
|
|
257
|
+
const scores = documents.map((d) => d.score);
|
|
258
|
+
const sortedScores = [...scores].sort((a, b) => a - b);
|
|
259
|
+
const median = sortedScores[Math.floor(sortedScores.length / 2)];
|
|
260
|
+
const threshold = median * 0.6;
|
|
261
|
+
let filtered = documents.filter((d) => d.score >= threshold);
|
|
262
|
+
// Step 2: Deduplicate near-identical content (Jaccard similarity > 0.8)
|
|
263
|
+
const deduplicated = [];
|
|
264
|
+
for (const doc of filtered) {
|
|
265
|
+
const isDuplicate = deduplicated.some((existing) => this.jaccardSimilarity(existing.content, doc.content) > 0.8);
|
|
266
|
+
if (!isDuplicate) {
|
|
267
|
+
deduplicated.push(doc);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
filtered = deduplicated;
|
|
271
|
+
// Step 3: LLM-based compression if available
|
|
272
|
+
if (this.config.llmFunction && filtered.length > 5) {
|
|
273
|
+
try {
|
|
274
|
+
const contextBlock = filtered.map((d, i) => `[${i}] ${d.content}`).join('\n\n');
|
|
275
|
+
const compressPrompt = `Given the query: "${query}"\n\n` +
|
|
276
|
+
`Below are retrieved passages. Return ONLY the indices (comma-separated) of the ` +
|
|
277
|
+
`passages that are most relevant and non-redundant. Return at most 5 indices.\n\n` +
|
|
278
|
+
contextBlock;
|
|
279
|
+
const response = await this.config.llmFunction(compressPrompt);
|
|
280
|
+
const indices = response
|
|
281
|
+
.replace(/[^0-9,]/g, '')
|
|
282
|
+
.split(',')
|
|
283
|
+
.map(Number)
|
|
284
|
+
.filter((n) => !isNaN(n) && n >= 0 && n < filtered.length);
|
|
285
|
+
if (indices.length > 0) {
|
|
286
|
+
return indices.map((i) => filtered[i]);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// Fallback to score-based top-5 below
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return filtered.slice(0, 5);
|
|
99
294
|
}
|
|
100
295
|
/**
|
|
101
|
-
*
|
|
296
|
+
* Jaccard similarity between two strings (token-level).
|
|
297
|
+
*/
|
|
298
|
+
jaccardSimilarity(a, b) {
|
|
299
|
+
const tokensA = new Set(a.toLowerCase().split(/\s+/));
|
|
300
|
+
const tokensB = new Set(b.toLowerCase().split(/\s+/));
|
|
301
|
+
const intersection = new Set([...tokensA].filter((t) => tokensB.has(t)));
|
|
302
|
+
const union = new Set([...tokensA, ...tokensB]);
|
|
303
|
+
return union.size === 0 ? 0 : intersection.size / union.size;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Self-query with automatic metadata extraction.
|
|
307
|
+
* Extracts filter conditions from natural language when an LLM is available,
|
|
308
|
+
* then applies them as metadata filters on the vector search.
|
|
102
309
|
*/
|
|
103
310
|
async selfQuery(naturalLanguageQuery) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
311
|
+
if (!this.config.llmFunction) {
|
|
312
|
+
return this.search(naturalLanguageQuery);
|
|
313
|
+
}
|
|
314
|
+
try {
|
|
315
|
+
const extractionPrompt = `Given the following natural language query, extract:\n` +
|
|
316
|
+
`1. A simplified search query (just the semantic intent)\n` +
|
|
317
|
+
`2. Any metadata filters expressed as JSON (e.g. {"category": "science", "year": 2024})\n\n` +
|
|
318
|
+
`Query: "${naturalLanguageQuery}"\n\n` +
|
|
319
|
+
`Respond in this exact JSON format:\n` +
|
|
320
|
+
`{"searchQuery": "...", "filters": {...}}`;
|
|
321
|
+
const response = await this.config.llmFunction(extractionPrompt);
|
|
322
|
+
const cleaned = response
|
|
323
|
+
.replace(/^```(?:json)?\s*/i, '')
|
|
324
|
+
.replace(/\s*```$/i, '')
|
|
325
|
+
.trim();
|
|
326
|
+
const parsed = JSON.parse(cleaned);
|
|
327
|
+
const searchQuery = parsed.searchQuery || naturalLanguageQuery;
|
|
328
|
+
const filter = parsed.filters && Object.keys(parsed.filters).length > 0 ? parsed.filters : undefined;
|
|
329
|
+
return this.search(searchQuery, { filter });
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
// Fallback to plain search on parse error
|
|
333
|
+
return this.search(naturalLanguageQuery);
|
|
334
|
+
}
|
|
107
335
|
}
|
|
108
336
|
/**
|
|
109
|
-
* Conversational RAG with session memory
|
|
337
|
+
* Conversational RAG with session memory.
|
|
338
|
+
* Maintains per-session conversation history and rewrites the user's
|
|
339
|
+
* follow-up question into a standalone query before retrieval.
|
|
110
340
|
*/
|
|
111
|
-
async chat(message,
|
|
112
|
-
|
|
113
|
-
|
|
341
|
+
async chat(message, sessionId) {
|
|
342
|
+
dbg('chat message=%s session=%s', message, sessionId);
|
|
343
|
+
// Get or create conversation history
|
|
344
|
+
if (!this.conversationHistory.has(sessionId)) {
|
|
345
|
+
this.conversationHistory.set(sessionId, []);
|
|
346
|
+
}
|
|
347
|
+
const history = this.conversationHistory.get(sessionId);
|
|
348
|
+
// Rewrite the follow-up into a standalone question using conversation context
|
|
349
|
+
let standaloneQuery = message;
|
|
350
|
+
if (history.length > 0 && this.config.llmFunction) {
|
|
351
|
+
try {
|
|
352
|
+
const historyText = history
|
|
353
|
+
.slice(-6) // last 3 exchanges
|
|
354
|
+
.map((m) => `${m.role}: ${m.content}`)
|
|
355
|
+
.join('\n');
|
|
356
|
+
const rewritePrompt = `Given the conversation history and a follow-up question, ` +
|
|
357
|
+
`rewrite the follow-up into a standalone question that captures the full context.\n\n` +
|
|
358
|
+
`Conversation:\n${historyText}\n\n` +
|
|
359
|
+
`Follow-up: ${message}\n\n` +
|
|
360
|
+
`Standalone question:`;
|
|
361
|
+
standaloneQuery = await this.config.llmFunction(rewritePrompt);
|
|
362
|
+
dbg('chat rewritten query=%s', standaloneQuery);
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
standaloneQuery = message;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
// Retrieve and generate
|
|
369
|
+
const sources = await this.retrieve(standaloneQuery);
|
|
370
|
+
const answer = await this.generate(standaloneQuery, sources);
|
|
371
|
+
// Update conversation history
|
|
372
|
+
history.push({ role: 'user', content: message });
|
|
373
|
+
history.push({ role: 'assistant', content: answer });
|
|
374
|
+
// Keep history bounded
|
|
375
|
+
while (history.length > 20) {
|
|
376
|
+
history.shift();
|
|
377
|
+
}
|
|
378
|
+
dbg('chat complete answer_len=%d sources=%d history_len=%d', answer.length, sources.length, history.length);
|
|
379
|
+
return { answer, sources };
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Clear conversation history for a session.
|
|
383
|
+
*/
|
|
384
|
+
clearChat(sessionId) {
|
|
385
|
+
this.conversationHistory.delete(sessionId);
|
|
114
386
|
}
|
|
115
387
|
/**
|
|
116
388
|
* Hybrid search combining vector and keyword search
|
|
@@ -119,19 +391,109 @@ let RAGService = class RAGService {
|
|
|
119
391
|
return this.search(query, { ...options, strategy: types_1.RetrievalStrategy.HYBRID });
|
|
120
392
|
}
|
|
121
393
|
/**
|
|
122
|
-
* Rerank search results
|
|
394
|
+
* Rerank search results using cross-encoder-style LLM scoring.
|
|
395
|
+
* When no LLM is configured, falls back to BM25-boosted re-scoring.
|
|
123
396
|
*/
|
|
124
|
-
async rerank(results,
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
397
|
+
async rerank(results, query, topN) {
|
|
398
|
+
const n = topN || 5;
|
|
399
|
+
if (results.length <= n)
|
|
400
|
+
return results;
|
|
401
|
+
// Strategy 1: LLM-based reranking (when available)
|
|
402
|
+
if (this.config.llmFunction) {
|
|
403
|
+
try {
|
|
404
|
+
const passages = results.map((r, i) => `[${i}] ${r.content.slice(0, 300)}`).join('\n\n');
|
|
405
|
+
const rerankPrompt = `Given the query: "${query}"\n\n` +
|
|
406
|
+
`Rank the following passages by relevance. Return ONLY the indices ` +
|
|
407
|
+
`in order from most to least relevant (comma-separated). ` +
|
|
408
|
+
`Return at most ${n} indices.\n\n${passages}`;
|
|
409
|
+
const response = await this.config.llmFunction(rerankPrompt);
|
|
410
|
+
const indices = response
|
|
411
|
+
.replace(/[^0-9,]/g, '')
|
|
412
|
+
.split(',')
|
|
413
|
+
.map(Number)
|
|
414
|
+
.filter((idx) => !isNaN(idx) && idx >= 0 && idx < results.length);
|
|
415
|
+
if (indices.length > 0) {
|
|
416
|
+
const seen = new Set();
|
|
417
|
+
const reranked = [];
|
|
418
|
+
for (const idx of indices) {
|
|
419
|
+
if (!seen.has(idx)) {
|
|
420
|
+
seen.add(idx);
|
|
421
|
+
reranked.push(results[idx]);
|
|
422
|
+
}
|
|
423
|
+
if (reranked.length >= n)
|
|
424
|
+
break;
|
|
425
|
+
}
|
|
426
|
+
return reranked;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
catch {
|
|
430
|
+
// Fallback to BM25 boost below
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
// Strategy 2: BM25 keyword boost
|
|
434
|
+
const bm25 = new bm25_1.BM25();
|
|
435
|
+
const bm25Docs = results.map((r) => ({
|
|
436
|
+
id: r.id,
|
|
437
|
+
content: r.content,
|
|
438
|
+
tokens: r.content
|
|
439
|
+
.toLowerCase()
|
|
440
|
+
.replace(/[^\w\s]/g, ' ')
|
|
441
|
+
.split(/\s+/)
|
|
442
|
+
.filter(Boolean),
|
|
443
|
+
}));
|
|
444
|
+
bm25.addDocuments(bm25Docs);
|
|
445
|
+
const keywordScores = bm25.search(query, results.length);
|
|
446
|
+
const kwMap = new Map(keywordScores.map((k) => [k.id, k.score]));
|
|
447
|
+
// Normalize BM25 scores
|
|
448
|
+
const maxKw = Math.max(...keywordScores.map((k) => k.score), 1);
|
|
449
|
+
const boosted = results.map((r) => ({
|
|
450
|
+
...r,
|
|
451
|
+
score: r.score * 0.7 + ((kwMap.get(r.id) || 0) / maxKw) * 0.3,
|
|
452
|
+
}));
|
|
453
|
+
return boosted.sort((a, b) => b.score - a.score).slice(0, n);
|
|
128
454
|
}
|
|
129
455
|
/**
|
|
130
|
-
* Ensemble retrieval combining multiple
|
|
456
|
+
* Ensemble retrieval combining multiple retrieval strategies.
|
|
457
|
+
* Runs each strategy, normalizes scores, applies weights, and fuses results.
|
|
131
458
|
*/
|
|
132
|
-
async ensemble(query,
|
|
133
|
-
|
|
134
|
-
|
|
459
|
+
async ensemble(query, methods, weights) {
|
|
460
|
+
if (methods.length === 0)
|
|
461
|
+
return this.search(query);
|
|
462
|
+
// Default: equal weights
|
|
463
|
+
const w = weights && weights.length === methods.length
|
|
464
|
+
? weights
|
|
465
|
+
: methods.map(() => 1 / methods.length);
|
|
466
|
+
// Run all strategies in parallel
|
|
467
|
+
const allResults = await Promise.all(methods.map((strategy) => this.search(query, { strategy, topK: 20 })));
|
|
468
|
+
// Normalize and weight each result set
|
|
469
|
+
const scoreMap = new Map();
|
|
470
|
+
for (let i = 0; i < allResults.length; i++) {
|
|
471
|
+
const results = allResults[i];
|
|
472
|
+
const weight = w[i];
|
|
473
|
+
// Min-max normalize within this strategy
|
|
474
|
+
const scores = results.map((r) => r.score);
|
|
475
|
+
const min = Math.min(...scores, 0);
|
|
476
|
+
const max = Math.max(...scores, 1);
|
|
477
|
+
const range = max - min || 1;
|
|
478
|
+
for (const result of results) {
|
|
479
|
+
const normalizedScore = ((result.score - min) / range) * weight;
|
|
480
|
+
const existing = scoreMap.get(result.id);
|
|
481
|
+
if (existing) {
|
|
482
|
+
existing.score += normalizedScore;
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
scoreMap.set(result.id, {
|
|
486
|
+
result: { ...result },
|
|
487
|
+
score: normalizedScore,
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
// Sort by fused score
|
|
493
|
+
const fused = [...scoreMap.values()]
|
|
494
|
+
.sort((a, b) => b.score - a.score)
|
|
495
|
+
.map(({ result, score }) => ({ ...result, score }));
|
|
496
|
+
return fused.slice(0, 10);
|
|
135
497
|
}
|
|
136
498
|
/**
|
|
137
499
|
* Time-weighted retrieval favoring recent documents
|