claude-local-docs 1.0.2 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +25 -5
- package/.claude-plugin/plugin.json +25 -4
- package/.mcp.json +9 -3
- package/README.md +139 -77
- package/commands/fetch-docs.md +146 -33
- package/dist/discovery.d.ts +46 -0
- package/dist/discovery.js +357 -0
- package/dist/discovery.js.map +1 -0
- package/dist/fetcher.d.ts +6 -1
- package/dist/fetcher.js +8 -5
- package/dist/fetcher.js.map +1 -1
- package/dist/index.js +85 -5
- package/dist/index.js.map +1 -1
- package/dist/indexer.d.ts +4 -4
- package/dist/indexer.js +95 -37
- package/dist/indexer.js.map +1 -1
- package/dist/integration.test.d.ts +8 -0
- package/dist/integration.test.js +114 -0
- package/dist/integration.test.js.map +1 -0
- package/dist/reranker.d.ts +2 -4
- package/dist/reranker.js +14 -42
- package/dist/reranker.js.map +1 -1
- package/dist/search.js +5 -4
- package/dist/search.js.map +1 -1
- package/docker-compose.nvidia.yml +14 -0
- package/docker-compose.yml +20 -0
- package/hooks/hooks.json +16 -0
- package/package.json +14 -3
- package/scripts/ensure-tei.sh +71 -0
- package/start-tei.sh +239 -0
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAG/C,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;AACzC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;AAExC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,OAAO;CACjB,CAAC,CAAC;AAEH,uCAAuC;AACvC,MAAM,CAAC,YAAY,CACjB,sBAAsB,EACtB;IACE,WAAW,EACT,yOAAyO;IAC3O,WAAW,EAAE;QACX,eAAe,EAAE,CAAC;aACf,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,qEAAqE,CAAC;KACnF;CACF,EACD,KAAK,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE;IAC5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,eAAe;YAC1B,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,cAAc,CAAC;gBACxC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC;gBAC7B,CAAC,CAAC,eAAe;YACnB,CAAC,CAAC,WAAW,CAAC;QAEhB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAE/C,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC5E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iCAAiC,GAAG,CAAC,OAAO,EAAE,EAAE;aAChF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,sCAAsC;AACtC,MAAM,CAAC,YAAY,CACjB,qBAAqB,EACrB;IACE,WAAW,EAAE,wJAAwJ;IACrK,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QAC9E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QAClE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;KACvE;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;IACjD,IAAI,CAAC;QACH,eAAe;QACf,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEzC,kBAAkB;QAClB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAErD,mBAAmB;QACnB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAE3E,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,IAAI;wBACb,OAAO;wBACP,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,cAAc,EAAE,MAAM,CAAC,SAAS;wBAChC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE;qBAC7B,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,kBAAkB,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACvF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,8BAA8B;AAC9B,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;IACE,WAAW,EAAE,mLAAmL;IAChM,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC1C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;QAC/E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;KAClF;CACF,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE;IACjC,IAAI,CAAC;QACH,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,sDAAsD;qBAC7D;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EACL,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG;gBACpB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;gBACjC,CAAC,CAAC,CAAC,CAAC,OAAO;SAChB,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC/E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,4BAA4B;AAC5B,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;IACE,WAAW,EAAE,wDAAwD;CACtE,EACD,KAAK,IAAI,EAAE;IACT,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,sDAAsD;qBAC7D;iBACF;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;aAC7E;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,uBAAuB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YAChF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,kCAAkC;AAClC,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;IACE,WAAW,EAAE,yEAAyE;IACtF,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QACzF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;KACzE;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE;IACtC,IAAI,CAAC;QACH,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,SAAS,OAAO,YAAY,EAAE,CAAC;iBACzE,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;4BACE,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,OAAO,EAAE,KAAK,CAAC,OAAO;4BACtB,WAAW,EAAE,KAAK,CAAC,WAAW;4BAC9B,OAAO,EAAE,KAAK,CAAC,IAAI;yBACpB,EACD,IAAI,EACJ,CAAC,CACF;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,gCAAgC,OAAO,4BAA4B,OAAO,GAAG;yBACpF;qBACF;iBACF,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,OAAO,EAAE,CAAC,CAAC,IAAI;aAChB,CAAC,CAAC,CAAC;YACJ,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;aAC/E,CAAC;QACJ,CAAC;QAED,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,uCAAuC,OAAO,GAAG;qBACxD;iBACF;aACF,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACtC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,OAAO,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;gBACxC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;gBACvC,CAAC,CAAC,CAAC,CAAC,WAAW;YACjB,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACnE,CAAC,CAAC,CAAC;QACJ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC7E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACnE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,sCAAsC;AACtC,MAAM,CAAC,YAAY,CACjB,qBAAqB,EACrB;IACE,WAAW,EACT,gMAAgM;IAClM,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QAC9E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC;KACrF;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,oBAAoB;QACpB,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC;qBACnE;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,eAAe;QACf,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QAErD,kBAAkB;QAClB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEjE,mBAAmB;QACnB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAErE,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,IAAI;wBACb,OAAO;wBACP,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,UAAU,EAAE,WAAW,CAAC,UAAU;wBAClC,cAAc,EAAE,MAAM,CAAC,SAAS;wBAChC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE;qBAC7B,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;iBAC7D;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,uBAAuB;AACvB,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AAEjF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;AAEzC;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAChC,OAAuB,EACvB,KAAe;IAEf,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAmB,EAAE,CAAC;IAEpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,qBAAqB;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE3B,iBAAiB;QACjB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACZ,GAAG,MAAM;YACT,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO;SAChE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;AAExC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;IAC3B,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,OAAO;CACjB,CAAC,CAAC;AAEH,uCAAuC;AACvC,MAAM,CAAC,YAAY,CACjB,sBAAsB,EACtB;IACE,WAAW,EACT,yOAAyO;IAC3O,WAAW,EAAE;QACX,eAAe,EAAE,CAAC;aACf,MAAM,EAAE;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,qEAAqE,CAAC;KACnF;CACF,EACD,KAAK,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE;IAC5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,eAAe;YAC1B,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,cAAc,CAAC;gBACxC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC;gBAC7B,CAAC,CAAC,eAAe;YACnB,CAAC,CAAC,WAAW,CAAC;QAEhB,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAE/C,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC5E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iCAAiC,GAAG,CAAC,OAAO,EAAE,EAAE;aAChF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,sCAAsC;AACtC,MAAM,CAAC,YAAY,CACjB,qBAAqB,EACrB;IACE,WAAW,EAAE,wJAAwJ;IACrK,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QAC9E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QAClE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;KACvE;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;IACjD,IAAI,CAAC;QACH,eAAe;QACf,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEzC,kBAAkB;QAClB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAErD,mBAAmB;QACnB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAE3E,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,IAAI;wBACb,OAAO;wBACP,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,cAAc,EAAE,MAAM,CAAC,SAAS;wBAChC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE;qBAC7B,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,kBAAkB,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACvF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,8BAA8B;AAC9B,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;IACE,WAAW,EAAE,mLAAmL;IAChM,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC1C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;QAC/E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;KAClF;CACF,EACD,KAAK,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE;IACjC,IAAI,CAAC;QACH,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1B,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,sDAAsD;qBAC7D;iBACF;aACF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,yDAAyD;QACzD,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE3D,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;QAEJ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC/E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,iBAAiB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1E,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,4BAA4B;AAC5B,MAAM,CAAC,YAAY,CACjB,WAAW,EACX;IACE,WAAW,EAAE,wDAAwD;CACtE,EACD,KAAK,IAAI,EAAE;IACT,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;QAC5C,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,sDAAsD;qBAC7D;iBACF;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;aAC7E;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,uBAAuB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YAChF,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,kCAAkC;AAClC,MAAM,CAAC,YAAY,CACjB,iBAAiB,EACjB;IACE,WAAW,EAAE,yEAAyE;IACtF,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QACzF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;KACzE;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE;IACtC,IAAI,CAAC;QACH,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,SAAS,OAAO,YAAY,EAAE,CAAC;iBACzE,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;4BACE,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,OAAO,EAAE,KAAK,CAAC,OAAO;4BACtB,WAAW,EAAE,KAAK,CAAC,WAAW;4BAC9B,OAAO,EAAE,KAAK,CAAC,IAAI;yBACpB,EACD,IAAI,EACJ,CAAC,CACF;qBACF;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,gCAAgC,OAAO,4BAA4B,OAAO,GAAG;yBACpF;qBACF;iBACF,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,OAAO,EAAE,CAAC,CAAC,IAAI;aAChB,CAAC,CAAC,CAAC;YACJ,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;aAC/E,CAAC;QACJ,CAAC;QAED,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,uCAAuC,OAAO,GAAG;qBACxD;iBACF;aACF,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YACtC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,OAAO,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ;gBACxC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;gBACvC,CAAC,CAAC,CAAC,CAAC,WAAW;YACjB,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACnE,CAAC,CAAC,CAAC;QACJ,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;SAC7E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACnE,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,sCAAsC;AACtC,MAAM,CAAC,YAAY,CACjB,qBAAqB,EACrB;IACE,WAAW,EACT,mMAAmM;IACrM,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gDAAgD,CAAC;QAC9E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0DAA0D,CAAC;KACrF;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE;IAClC,IAAI,CAAC;QACH,oBAAoB;QACpB,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAC;qBACnE;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,eAAe;QACf,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;QAErD,kBAAkB;QAClB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEjE,mBAAmB;QACnB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAErE,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,IAAI;wBACb,OAAO;wBACP,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,UAAU,EAAE,WAAW,CAAC,UAAU;wBAClC,cAAc,EAAE,MAAM,CAAC,SAAS;wBAChC,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE;qBAC7B,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;iBAC7D;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,0CAA0C;AAC1C,MAAM,CAAC,YAAY,CACjB,yBAAyB,EACzB;IACE,WAAW,EACT,qWAAqW;IACvW,WAAW,EAAE;QACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;QAClF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;KAC9F;CACF,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE;IAC7B,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAE3F,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,IAAI;wBACb,OAAO;wBACP,MAAM,EAAE,SAAS,CAAC,MAAM;wBACxB,GAAG,EAAE,SAAS,CAAC,GAAG;wBAClB,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,UAAU,EAAE,SAAS,CAAC,UAAU;wBAChC,cAAc,EAAE,MAAM,CAAC,SAAS;wBAChC,YAAY,EAAE,SAAS,CAAC,YAAY;wBACpC,UAAU,EAAE,SAAS,CAAC,UAAU;wBAChC,OAAO,EAAE,SAAS,CAAC,OAAO;qBAC3B,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;iBACtE;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC,CACF,CAAC;AAEF,uBAAuB;AACvB,KAAK,UAAU,IAAI;IACjB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/indexer.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { DocRow } from "./types.js";
|
|
2
2
|
/**
|
|
3
3
|
* Semantic chunking: split markdown by headings, then split large sections
|
|
4
|
-
* at paragraph boundaries
|
|
4
|
+
* at paragraph boundaries. Code fences are never split.
|
|
5
5
|
*/
|
|
6
6
|
export declare function chunkMarkdown(markdown: string, library: string): Omit<DocRow, "id" | "vector">[];
|
|
7
7
|
/**
|
|
8
|
-
* Generate embeddings
|
|
9
|
-
* Uses Matryoshka dimensionality reduction to
|
|
10
|
-
* Prefixes text with task type for better accuracy.
|
|
8
|
+
* Generate embeddings via TEI (Text Embeddings Inference).
|
|
9
|
+
* Uses Matryoshka dimensionality reduction to 384 dims + L2 normalize.
|
|
10
|
+
* Prefixes text with task type for better accuracy (required by nomic).
|
|
11
11
|
*/
|
|
12
12
|
export declare function embedTexts(texts: string[], taskType?: "search_document" | "search_query"): Promise<number[][]>;
|
|
13
13
|
/**
|
package/dist/indexer.js
CHANGED
|
@@ -1,22 +1,49 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
1
|
+
const TEI_EMBED_URL = process.env.TEI_EMBED_URL ?? "http://localhost:39281";
|
|
2
|
+
const MATRYOSHKA_DIM = 384;
|
|
3
|
+
// ── Code fence detection ────────────────────────────────────────────────
|
|
4
|
+
const CODE_FENCE_OPEN = /^(`{3,}|~{3,})/;
|
|
5
|
+
const CODE_FENCE_CLOSE = (fence) => new RegExp(`^${fence.replace(/~/g, "\\~")}\\s*$`);
|
|
6
|
+
/**
|
|
7
|
+
* Pre-process markdown: merge each fenced code block into a single "line"
|
|
8
|
+
* so that paragraph-based splitting never breaks inside code.
|
|
9
|
+
* The merged line uses \n internally but won't be split by \n\n.
|
|
10
|
+
*/
|
|
11
|
+
function mergeCodeFences(lines) {
|
|
12
|
+
const result = [];
|
|
13
|
+
let i = 0;
|
|
14
|
+
while (i < lines.length) {
|
|
15
|
+
const fenceMatch = lines[i].match(CODE_FENCE_OPEN);
|
|
16
|
+
if (fenceMatch) {
|
|
17
|
+
// Collect everything inside the fence
|
|
18
|
+
const fence = fenceMatch[1];
|
|
19
|
+
const closePat = CODE_FENCE_CLOSE(fence);
|
|
20
|
+
const block = [lines[i]];
|
|
21
|
+
i++;
|
|
22
|
+
while (i < lines.length && !closePat.test(lines[i].trim())) {
|
|
23
|
+
block.push(lines[i]);
|
|
24
|
+
i++;
|
|
25
|
+
}
|
|
26
|
+
if (i < lines.length) {
|
|
27
|
+
block.push(lines[i]); // closing fence
|
|
28
|
+
i++;
|
|
29
|
+
}
|
|
30
|
+
// Join with a sentinel that won't match \n\n splitting
|
|
31
|
+
result.push(block.join("\n"));
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
result.push(lines[i]);
|
|
35
|
+
i++;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return result;
|
|
13
39
|
}
|
|
14
40
|
/**
|
|
15
41
|
* Semantic chunking: split markdown by headings, then split large sections
|
|
16
|
-
* at paragraph boundaries
|
|
42
|
+
* at paragraph boundaries. Code fences are never split.
|
|
17
43
|
*/
|
|
18
44
|
export function chunkMarkdown(markdown, library) {
|
|
19
|
-
const
|
|
45
|
+
const rawLines = markdown.split("\n");
|
|
46
|
+
const lines = mergeCodeFences(rawLines);
|
|
20
47
|
const chunks = [];
|
|
21
48
|
const headingStack = [];
|
|
22
49
|
let currentText = "";
|
|
@@ -24,19 +51,24 @@ export function chunkMarkdown(markdown, library) {
|
|
|
24
51
|
const trimmed = currentText.trim();
|
|
25
52
|
if (trimmed.length > 0) {
|
|
26
53
|
const headingPath = headingStack.map((h) => h.text);
|
|
27
|
-
|
|
54
|
+
// Prepend heading context so BM25 can match on section structure
|
|
55
|
+
// (e.g., "Connect > Onboarding" makes "connect onboarding" query match)
|
|
56
|
+
const headingPrefix = headingPath.length > 0 ? `[${headingPath.join(" > ")}]\n\n` : "";
|
|
57
|
+
const subChunks = splitWithOverlap(trimmed, 4000, 400);
|
|
28
58
|
for (const sub of subChunks) {
|
|
29
59
|
chunks.push({
|
|
30
60
|
library,
|
|
31
61
|
headingPath: JSON.stringify(headingPath),
|
|
32
|
-
text: sub,
|
|
62
|
+
text: headingPrefix + sub,
|
|
33
63
|
});
|
|
34
64
|
}
|
|
35
65
|
}
|
|
36
66
|
currentText = "";
|
|
37
67
|
}
|
|
38
68
|
for (const line of lines) {
|
|
39
|
-
|
|
69
|
+
// Only match headings on non-code lines (code fences contain \n)
|
|
70
|
+
const isCodeBlock = line.includes("\n");
|
|
71
|
+
const headingMatch = !isCodeBlock ? line.match(/^(#{1,6})\s+(.+)/) : null;
|
|
40
72
|
if (headingMatch) {
|
|
41
73
|
flush();
|
|
42
74
|
const level = headingMatch[1].length;
|
|
@@ -58,7 +90,7 @@ export function chunkMarkdown(markdown, library) {
|
|
|
58
90
|
}
|
|
59
91
|
/**
|
|
60
92
|
* Split text at paragraph boundaries with ~10% overlap.
|
|
61
|
-
*
|
|
93
|
+
* Code blocks (containing \n) are treated as atomic units.
|
|
62
94
|
*/
|
|
63
95
|
function splitWithOverlap(text, maxChars, overlapChars) {
|
|
64
96
|
if (text.length <= maxChars)
|
|
@@ -66,12 +98,20 @@ function splitWithOverlap(text, maxChars, overlapChars) {
|
|
|
66
98
|
const paragraphs = text.split(/\n\n+/);
|
|
67
99
|
const result = [];
|
|
68
100
|
let current = "";
|
|
69
|
-
let overlapBuffer = "";
|
|
101
|
+
let overlapBuffer = "";
|
|
70
102
|
for (const para of paragraphs) {
|
|
71
|
-
|
|
103
|
+
const nextLen = current.length + para.length + 2;
|
|
104
|
+
// If adding this paragraph would exceed limit AND we have content already
|
|
105
|
+
if (nextLen > maxChars && current.length > 0) {
|
|
106
|
+
// But if the paragraph is a code block, try to keep it with the heading before it
|
|
107
|
+
// by allowing a larger chunk rather than splitting before the code
|
|
108
|
+
if (para.includes("\n") && current.length < maxChars * 0.6) {
|
|
109
|
+
// Current chunk is less than 60% full — keep the code block with it
|
|
110
|
+
current += (current ? "\n\n" : "") + para;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
72
113
|
result.push(current.trim());
|
|
73
|
-
|
|
74
|
-
overlapBuffer = current.slice(-overlapChars);
|
|
114
|
+
overlapBuffer = snapOverlapToParagraph(current, overlapChars);
|
|
75
115
|
current = overlapBuffer + "\n\n" + para;
|
|
76
116
|
}
|
|
77
117
|
else {
|
|
@@ -84,26 +124,44 @@ function splitWithOverlap(text, maxChars, overlapChars) {
|
|
|
84
124
|
return result;
|
|
85
125
|
}
|
|
86
126
|
/**
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
127
|
+
* Walk backward through paragraph breaks to find the last complete
|
|
128
|
+
* paragraph(s) fitting within maxChars. Falls back to raw char slice
|
|
129
|
+
* if no paragraph breaks exist in the tail.
|
|
130
|
+
*/
|
|
131
|
+
function snapOverlapToParagraph(text, maxChars) {
|
|
132
|
+
const tail = text.slice(-maxChars);
|
|
133
|
+
const breakIdx = tail.indexOf("\n\n");
|
|
134
|
+
if (breakIdx === -1) {
|
|
135
|
+
return tail;
|
|
136
|
+
}
|
|
137
|
+
return tail.slice(breakIdx + 2);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Generate embeddings via TEI (Text Embeddings Inference).
|
|
141
|
+
* Uses Matryoshka dimensionality reduction to 384 dims + L2 normalize.
|
|
142
|
+
* Prefixes text with task type for better accuracy (required by nomic).
|
|
90
143
|
*/
|
|
91
144
|
export async function embedTexts(texts, taskType = "search_document") {
|
|
92
|
-
const { pipe, layerNorm } = await getEmbedPipeline();
|
|
93
|
-
const embeddings = [];
|
|
94
|
-
// Prefix each text with the task type as required by nomic
|
|
95
145
|
const prefixed = texts.map((t) => `${taskType}: ${t}`);
|
|
96
|
-
|
|
97
|
-
|
|
146
|
+
const embeddings = [];
|
|
147
|
+
// Process in batches to stay within TEI request size limits
|
|
148
|
+
const batchSize = 32;
|
|
98
149
|
for (let i = 0; i < prefixed.length; i += batchSize) {
|
|
99
150
|
const batch = prefixed.slice(i, i + batchSize);
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
151
|
+
const res = await fetch(`${TEI_EMBED_URL}/embed`, {
|
|
152
|
+
method: "POST",
|
|
153
|
+
headers: { "Content-Type": "application/json" },
|
|
154
|
+
body: JSON.stringify({ inputs: batch, truncate: true }),
|
|
155
|
+
});
|
|
156
|
+
if (!res.ok) {
|
|
157
|
+
throw new Error(`TEI embed error: ${res.status} ${await res.text()}`);
|
|
158
|
+
}
|
|
159
|
+
const fullVecs = await res.json();
|
|
160
|
+
// Matryoshka truncation to 384 dims + L2 normalize
|
|
161
|
+
for (const vec of fullVecs) {
|
|
162
|
+
const truncated = vec.slice(0, MATRYOSHKA_DIM);
|
|
163
|
+
const norm = Math.sqrt(truncated.reduce((s, v) => s + v * v, 0));
|
|
164
|
+
embeddings.push(truncated.map((v) => v / (norm || 1)));
|
|
107
165
|
}
|
|
108
166
|
}
|
|
109
167
|
return embeddings;
|
package/dist/indexer.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAEA,
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAEA,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,wBAAwB,CAAC;AAC5E,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,2EAA2E;AAE3E,MAAM,eAAe,GAAG,gBAAgB,CAAC;AACzC,MAAM,gBAAgB,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;AAE9F;;;;GAIG;AACH,SAAS,eAAe,CAAC,KAAe;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACnD,IAAI,UAAU,EAAE,CAAC;YACf,sCAAsC;YACtC,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,KAAK,GAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC,EAAE,CAAC;YACJ,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;gBAC3D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB;gBACtC,CAAC,EAAE,CAAC;YACN,CAAC;YACD,uDAAuD;YACvD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAgB,EAChB,OAAe;IAEf,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,MAAM,GAAoC,EAAE,CAAC;IAEnD,MAAM,YAAY,GAAsC,EAAE,CAAC;IAC3D,IAAI,WAAW,GAAG,EAAE,CAAC;IAErB,SAAS,KAAK;QACZ,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpD,iEAAiE;YACjE,wEAAwE;YACxE,MAAM,aAAa,GACjB,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;YACvD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;oBACxC,IAAI,EAAE,aAAa,GAAG,GAAG;iBAC1B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,WAAW,GAAG,EAAE,CAAC;IACnB,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,iEAAiE;QACjE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,YAAY,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,IAAI,YAAY,EAAE,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAEpC,uCAAuC;YACvC,OACE,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,EACpD,CAAC;gBACD,YAAY,CAAC,GAAG,EAAE,CAAC;YACrB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACnC,WAAW,IAAI,IAAI,GAAG,IAAI,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,WAAW,IAAI,IAAI,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,KAAK,EAAE,CAAC;IAER,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,QAAgB,EAChB,YAAoB;IAEpB,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,aAAa,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAEjD,0EAA0E;QAC1E,IAAI,OAAO,GAAG,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,kFAAkF;YAClF,mEAAmE;YACnE,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;gBAC3D,oEAAoE;gBACpE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;gBAC1C,SAAS;YACX,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5B,aAAa,GAAG,sBAAsB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YAC9D,OAAO,GAAG,aAAa,GAAG,MAAM,GAAG,IAAI,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QAC5C,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,IAAY,EAAE,QAAgB;IAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,WAA+C,iBAAiB;IAEhE,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;IACvD,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,4DAA4D;IAC5D,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAE/C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,aAAa,QAAQ,EAAE;YAChD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;SACxD,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,MAAM,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAe,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAE9C,mDAAmD;QACnD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACjE,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,OAAe;IAEf,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IAE9D,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,KAAK;QACR,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test: fetch Prisma llms-full.txt, index it, and run searches.
|
|
3
|
+
*
|
|
4
|
+
* Exercises the full fetch -> chunk -> embed -> store -> search -> rerank pipeline.
|
|
5
|
+
*
|
|
6
|
+
* Run: npm test
|
|
7
|
+
*/
|
|
8
|
+
import { describe, it, before, after } from "node:test";
|
|
9
|
+
import assert from "node:assert/strict";
|
|
10
|
+
import { mkdtemp, rm, readFile, writeFile, mkdir } from "node:fs/promises";
|
|
11
|
+
import { tmpdir } from "node:os";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { existsSync } from "node:fs";
|
|
14
|
+
import { fetchDocContent } from "./fetcher.js";
|
|
15
|
+
import { chunkMarkdown, indexDocument } from "./indexer.js";
|
|
16
|
+
import { DocStore } from "./store.js";
|
|
17
|
+
import { searchDocs } from "./search.js";
|
|
18
|
+
const PRISMA_URL = "https://www.prisma.io/docs/llms-full.txt";
|
|
19
|
+
const CACHE_DIR = join(tmpdir(), "claude-local-docs-fixture");
|
|
20
|
+
const CACHE_FILE = join(CACHE_DIR, "prisma-llms-full.txt");
|
|
21
|
+
let tempDir;
|
|
22
|
+
let store;
|
|
23
|
+
let fetchedContent;
|
|
24
|
+
/** Load from cache or fetch from network. */
|
|
25
|
+
async function loadPrismaDoc() {
|
|
26
|
+
if (existsSync(CACHE_FILE)) {
|
|
27
|
+
console.log(" Using cached Prisma docs");
|
|
28
|
+
return readFile(CACHE_FILE, "utf-8");
|
|
29
|
+
}
|
|
30
|
+
console.log(" Fetching Prisma docs (first run, will be cached)...");
|
|
31
|
+
const result = await fetchDocContent(PRISMA_URL);
|
|
32
|
+
assert.equal(result.ok, true, `Fetch failed: ${"error" in result ? result.error : ""}`);
|
|
33
|
+
assert.ok(result.ok);
|
|
34
|
+
await mkdir(CACHE_DIR, { recursive: true });
|
|
35
|
+
await writeFile(CACHE_FILE, result.content, "utf-8");
|
|
36
|
+
console.log(` Fetched and cached ${result.byteLength} bytes (${(result.byteLength / 1024 / 1024).toFixed(1)} MB)`);
|
|
37
|
+
return result.content;
|
|
38
|
+
}
|
|
39
|
+
describe("Prisma llms-full.txt integration", { timeout: 600_000 }, () => {
|
|
40
|
+
before(async () => {
|
|
41
|
+
tempDir = await mkdtemp(join(tmpdir(), "claude-local-docs-test-"));
|
|
42
|
+
fetchedContent = await loadPrismaDoc();
|
|
43
|
+
});
|
|
44
|
+
after(async () => {
|
|
45
|
+
if (tempDir && existsSync(tempDir)) {
|
|
46
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
// -- Fetch --
|
|
50
|
+
it("fetched content is substantial", () => {
|
|
51
|
+
assert.ok(fetchedContent.length > 1_000_000, `Expected > 1M chars, got ${fetchedContent.length}`);
|
|
52
|
+
});
|
|
53
|
+
// -- Chunking --
|
|
54
|
+
it("chunks the markdown into reasonable pieces", () => {
|
|
55
|
+
const chunks = chunkMarkdown(fetchedContent, "prisma");
|
|
56
|
+
console.log(` Produced ${chunks.length} chunks`);
|
|
57
|
+
assert.ok(chunks.length > 100, `Expected >100 chunks, got ${chunks.length}`);
|
|
58
|
+
for (const c of chunks) {
|
|
59
|
+
assert.equal(c.library, "prisma");
|
|
60
|
+
assert.ok(c.text.trim().length > 0, "Found empty chunk");
|
|
61
|
+
const parsed = JSON.parse(c.headingPath);
|
|
62
|
+
assert.ok(Array.isArray(parsed), `headingPath not an array: ${c.headingPath}`);
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
// -- Index + Store --
|
|
66
|
+
it("indexes and stores all chunks in LanceDB", async () => {
|
|
67
|
+
store = new DocStore(tempDir);
|
|
68
|
+
const chunks = await indexDocument(fetchedContent, "prisma");
|
|
69
|
+
console.log(` Indexed ${chunks.length} chunks with embeddings`);
|
|
70
|
+
assert.ok(chunks.length > 100);
|
|
71
|
+
for (const c of chunks) {
|
|
72
|
+
assert.equal(c.vector.length, 384, `Expected 384-dim vector, got ${c.vector.length}`);
|
|
73
|
+
}
|
|
74
|
+
const result = await store.addLibrary("prisma", "6.0.0", PRISMA_URL, chunks);
|
|
75
|
+
console.log(` Stored: ${result.chunkCount} chunks, index size: ${result.indexSize}`);
|
|
76
|
+
assert.equal(result.chunkCount, chunks.length);
|
|
77
|
+
assert.equal(result.indexSize, chunks.length);
|
|
78
|
+
});
|
|
79
|
+
// -- Search --
|
|
80
|
+
const searchCases = [
|
|
81
|
+
{ query: "prisma client query", expectTerms: ["prisma", "client", "query"] },
|
|
82
|
+
{ query: "database migration", expectTerms: ["migrat", "schema", "database"] },
|
|
83
|
+
{ query: "prisma schema relations", expectTerms: ["relation", "schema", "model"] },
|
|
84
|
+
];
|
|
85
|
+
for (const { query, expectTerms } of searchCases) {
|
|
86
|
+
it(`returns relevant results for '${query}'`, async () => {
|
|
87
|
+
const results = await searchDocs(query, store, { library: "prisma", topK: 5 });
|
|
88
|
+
console.log(` '${query}' → top score=${results[0]?.score}, heading=${results[0]?.headingPath.join(" > ")}`);
|
|
89
|
+
assert.ok(results.length > 0, "Expected at least 1 result");
|
|
90
|
+
assert.ok(results.length <= 5);
|
|
91
|
+
const topContent = results[0].content.toLowerCase();
|
|
92
|
+
assert.ok(expectTerms.some((t) => topContent.includes(t)), `Top result doesn't contain any of [${expectTerms}]`);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
it("returns scores in descending order", async () => {
|
|
96
|
+
const results = await searchDocs("CRUD operations", store, { library: "prisma", topK: 10 });
|
|
97
|
+
assert.ok(results.length > 1);
|
|
98
|
+
for (let i = 1; i < results.length; i++) {
|
|
99
|
+
assert.ok(results[i - 1].score >= results[i].score, `Not sorted: [${i - 1}]=${results[i - 1].score} < [${i}]=${results[i].score}`);
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
it("returns results with valid structure", async () => {
|
|
103
|
+
const results = await searchDocs("authentication", store, { topK: 3 });
|
|
104
|
+
for (const r of results) {
|
|
105
|
+
assert.equal(typeof r.score, "number");
|
|
106
|
+
assert.equal(typeof r.library, "string");
|
|
107
|
+
assert.ok(Array.isArray(r.headingPath));
|
|
108
|
+
assert.equal(typeof r.content, "string");
|
|
109
|
+
assert.equal(typeof r.chunkId, "number");
|
|
110
|
+
assert.ok(r.content.length > 0);
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
//# sourceMappingURL=integration.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"integration.test.js","sourceRoot":"","sources":["../src/integration.test.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AACxD,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAErC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,GAAG,0CAA0C,CAAC;AAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,2BAA2B,CAAC,CAAC;AAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;AAE3D,IAAI,OAAe,CAAC;AACpB,IAAI,KAAe,CAAC;AACpB,IAAI,cAAsB,CAAC;AAE3B,6CAA6C;AAC7C,KAAK,UAAU,aAAa;IAC1B,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,iBAAiB,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAErB,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,UAAU,WAAW,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpH,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,QAAQ,CAAC,kCAAkC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE;IACtE,MAAM,CAAC,KAAK,IAAI,EAAE;QAChB,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,yBAAyB,CAAC,CAAC,CAAC;QACnE,cAAc,GAAG,MAAM,aAAa,EAAE,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,IAAI,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACnC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,cAAc;IAEd,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,CAAC,EAAE,CAAC,cAAc,CAAC,MAAM,GAAG,SAAS,EAAE,4BAA4B,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;IACpG,CAAC,CAAC,CAAC;IAEH,iBAAiB;IAEjB,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAElD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,6BAA6B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAClC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,mBAAmB,CAAC,CAAC;YACzD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACzC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,6BAA6B,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,sBAAsB;IAEtB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE9B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,yBAAyB,CAAC,CAAC;QAEjE,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,gCAAgC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,UAAU,wBAAwB,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;QAEtF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,eAAe;IAEf,MAAM,WAAW,GAAG;QAClB,EAAE,KAAK,EAAE,qBAAqB,EAAE,WAAW,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QAC5E,EAAE,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE;QAC9E,EAAE,KAAK,EAAE,yBAAyB,EAAE,WAAW,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;KACnF,CAAC;IAEF,KAAK,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,WAAW,EAAE,CAAC;QACjD,EAAE,CAAC,iCAAiC,KAAK,GAAG,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/E,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,iBAAiB,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAE7G,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,4BAA4B,CAAC,CAAC;YAC5D,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;YAE/B,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YACpD,MAAM,CAAC,EAAE,CACP,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAC/C,sCAAsC,WAAW,GAAG,CACrD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,iBAAiB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5F,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,gBAAgB,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACrI,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,gBAAgB,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACzC,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACzC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/reranker.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Cross-encoder reranker
|
|
2
|
+
* Cross-encoder reranker via TEI (Text Embeddings Inference).
|
|
3
3
|
* Takes query + candidate passages, returns relevance scores.
|
|
4
|
-
* ~23MB model, runs locally via Transformers.js ONNX runtime.
|
|
5
4
|
*/
|
|
6
5
|
export interface RerankCandidate {
|
|
7
6
|
id: number;
|
|
@@ -14,7 +13,6 @@ export interface RerankResult extends RerankCandidate {
|
|
|
14
13
|
rerankerScore: number;
|
|
15
14
|
}
|
|
16
15
|
/**
|
|
17
|
-
* Rerank candidates using cross-encoder.
|
|
18
|
-
* Processes in batches to manage memory.
|
|
16
|
+
* Rerank candidates using TEI cross-encoder endpoint.
|
|
19
17
|
*/
|
|
20
18
|
export declare function rerank(query: string, candidates: RerankCandidate[]): Promise<RerankResult[]>;
|
package/dist/reranker.js
CHANGED
|
@@ -1,53 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Cross-encoder reranker
|
|
2
|
+
* Cross-encoder reranker via TEI (Text Embeddings Inference).
|
|
3
3
|
* Takes query + candidate passages, returns relevance scores.
|
|
4
|
-
* ~23MB model, runs locally via Transformers.js ONNX runtime.
|
|
5
4
|
*/
|
|
6
|
-
|
|
7
|
-
let rerankerTokenizer = null;
|
|
8
|
-
const RERANKER_MODEL = "Xenova/ms-marco-MiniLM-L-6-v2";
|
|
9
|
-
async function loadReranker() {
|
|
10
|
-
if (rerankerModel && rerankerTokenizer) {
|
|
11
|
-
return { model: rerankerModel, tokenizer: rerankerTokenizer };
|
|
12
|
-
}
|
|
13
|
-
const { AutoModelForSequenceClassification, AutoTokenizer } = await import("@huggingface/transformers");
|
|
14
|
-
rerankerModel = await AutoModelForSequenceClassification.from_pretrained(RERANKER_MODEL);
|
|
15
|
-
rerankerTokenizer = await AutoTokenizer.from_pretrained(RERANKER_MODEL);
|
|
16
|
-
return { model: rerankerModel, tokenizer: rerankerTokenizer };
|
|
17
|
-
}
|
|
5
|
+
const TEI_RERANK_URL = process.env.TEI_RERANK_URL ?? "http://localhost:39282";
|
|
18
6
|
/**
|
|
19
|
-
* Rerank candidates using cross-encoder.
|
|
20
|
-
* Processes in batches to manage memory.
|
|
7
|
+
* Rerank candidates using TEI cross-encoder endpoint.
|
|
21
8
|
*/
|
|
22
9
|
export async function rerank(query, candidates) {
|
|
23
10
|
if (candidates.length === 0)
|
|
24
11
|
return [];
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const passages = batch.map((c) => c.text);
|
|
33
|
-
const features = tokenizer(queries, {
|
|
34
|
-
text_pair: passages,
|
|
35
|
-
padding: true,
|
|
36
|
-
truncation: true,
|
|
37
|
-
});
|
|
38
|
-
const output = await model(features);
|
|
39
|
-
// output.logits is a Tensor of shape [batch_size, 1]
|
|
40
|
-
// Higher score = more relevant
|
|
41
|
-
for (let j = 0; j < batch.length; j++) {
|
|
42
|
-
const score = output.logits.data[j];
|
|
43
|
-
results.push({
|
|
44
|
-
...batch[j],
|
|
45
|
-
rerankerScore: score,
|
|
46
|
-
});
|
|
47
|
-
}
|
|
12
|
+
const res = await fetch(`${TEI_RERANK_URL}/rerank`, {
|
|
13
|
+
method: "POST",
|
|
14
|
+
headers: { "Content-Type": "application/json" },
|
|
15
|
+
body: JSON.stringify({ query, texts: candidates.map((c) => c.text) }),
|
|
16
|
+
});
|
|
17
|
+
if (!res.ok) {
|
|
18
|
+
throw new Error(`TEI rerank error: ${res.status} ${await res.text()}`);
|
|
48
19
|
}
|
|
49
|
-
|
|
50
|
-
results
|
|
51
|
-
|
|
20
|
+
const results = await res.json();
|
|
21
|
+
return results
|
|
22
|
+
.map((r) => ({ ...candidates[r.index], rerankerScore: r.score }))
|
|
23
|
+
.sort((a, b) => b.rerankerScore - a.rerankerScore);
|
|
52
24
|
}
|
|
53
25
|
//# sourceMappingURL=reranker.js.map
|
package/dist/reranker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,wBAAwB,CAAC;AAc9E;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,UAA6B;IAE7B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,cAAc,SAAS,EAAE;QAClD,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;KACtE,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,MAAM,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,OAAO,GAAuC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAErE,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;SAChE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;AACvD,CAAC"}
|
package/dist/search.js
CHANGED
|
@@ -46,7 +46,7 @@ function reciprocalRankFusion(vectorRanked, bm25Ranked, options) {
|
|
|
46
46
|
// --- Main search pipeline ---
|
|
47
47
|
export async function searchDocs(query, store, options) {
|
|
48
48
|
const topK = options?.topK ?? 10;
|
|
49
|
-
const candidateCount = 50; // retrieve
|
|
49
|
+
const candidateCount = 50; // retrieve enough for reranking without noise
|
|
50
50
|
// Step 1: Vector search via LanceDB
|
|
51
51
|
const [queryVector] = await embedTexts([query], "search_query");
|
|
52
52
|
const vectorHits = await store.vectorSearch(queryVector, candidateCount, options?.library);
|
|
@@ -64,15 +64,16 @@ export async function searchDocs(query, store, options) {
|
|
|
64
64
|
library: row.library,
|
|
65
65
|
headingPath: row.headingPath,
|
|
66
66
|
}));
|
|
67
|
-
// Step 3: RRF fusion (k=60, BM25
|
|
67
|
+
// Step 3: RRF fusion (k=60, BM25 weighted higher — trust exact keyword matches for
|
|
68
|
+
// framework-specific queries like "Next.js middleware" vs "NestJS middleware")
|
|
68
69
|
const fused = reciprocalRankFusion(vectorRanked, bm25Ranked, {
|
|
69
70
|
k: 60,
|
|
70
71
|
vectorWeight: 0.7,
|
|
71
72
|
bm25Weight: 1.0,
|
|
72
73
|
});
|
|
73
|
-
// Step 4: Cross-encoder rerank top
|
|
74
|
+
// Step 4: Cross-encoder rerank top 50 candidates
|
|
74
75
|
const rerankCandidates = fused
|
|
75
|
-
.slice(0,
|
|
76
|
+
.slice(0, 50)
|
|
76
77
|
.map((f) => ({
|
|
77
78
|
id: f.id,
|
|
78
79
|
text: f.text,
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAwB,MAAM,eAAe,CAAC;AAe7D;;;GAGG;AACH,SAAS,oBAAoB,CAC3B,YAAyB,EACzB,UAAuB,EACvB,OAAgE;IAEhE,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEhD,6CAA6C;IAC7C,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+BAA+B;AAE/B,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,KAAe,EACf,OAA6C;IAE7C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;IACjC,MAAM,cAAc,GAAG,EAAE,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAwB,MAAM,eAAe,CAAC;AAe7D;;;GAGG;AACH,SAAS,oBAAoB,CAC3B,YAAyB,EACzB,UAAuB,EACvB,OAAgE;IAEhE,MAAM,EAAE,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;IAEhD,6CAA6C;IAC7C,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,QAAQ,IAAI,KAAK,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+BAA+B;AAE/B,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,KAAe,EACf,OAA6C;IAE7C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;IACjC,MAAM,cAAc,GAAG,EAAE,CAAC,CAAC,8CAA8C;IAEzE,oCAAoC;IACpC,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,YAAY,CACzC,WAAW,EACX,cAAc,EACd,OAAO,EAAE,OAAO,CACjB,CAAC;IACF,MAAM,YAAY,GAAgB,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACzD,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC,CAAC,CAAC;IAEJ,6CAA6C;IAC7C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAgB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC;QAC5D,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC,CAAC,CAAC;IAEJ,mFAAmF;IACnF,+EAA+E;IAC/E,MAAM,KAAK,GAAG,oBAAoB,CAAC,YAAY,EAAE,UAAU,EAAE;QAC3D,CAAC,EAAE,EAAE;QACL,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;IAEH,iDAAiD;IACjD,MAAM,gBAAgB,GAAsB,KAAK;SAC9C,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC,CAAC,CAAC;IAEN,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC;IAEvD,kCAAkC;IAClC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,IAAI;QAChD,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAa;QAClD,OAAO,EAAE,CAAC,CAAC,IAAI;QACf,OAAO,EAAE,CAAC,CAAC,EAAE;KACd,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
services:
|
|
2
|
+
tei-embed:
|
|
3
|
+
image: ghcr.io/huggingface/text-embeddings-inference:${TEI_TAG:-cpu-1.9}
|
|
4
|
+
command: --model-id nomic-ai/nomic-embed-text-v1.5 --port 80 --max-client-batch-size 64
|
|
5
|
+
ports:
|
|
6
|
+
- "39281:80"
|
|
7
|
+
volumes:
|
|
8
|
+
- tei-embed-cache:/data
|
|
9
|
+
|
|
10
|
+
tei-rerank:
|
|
11
|
+
image: ghcr.io/huggingface/text-embeddings-inference:${TEI_TAG:-cpu-1.9}
|
|
12
|
+
command: --model-id cross-encoder/ms-marco-MiniLM-L-6-v2 --port 80
|
|
13
|
+
ports:
|
|
14
|
+
- "39282:80"
|
|
15
|
+
volumes:
|
|
16
|
+
- tei-rerank-cache:/data
|
|
17
|
+
|
|
18
|
+
volumes:
|
|
19
|
+
tei-embed-cache:
|
|
20
|
+
tei-rerank-cache:
|
package/hooks/hooks.json
ADDED