@staticn0va/wigolo 0.6.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -48
- package/SKILL.md +22 -22
- package/assets/skills/wigolo/rules/cache-first.md +1 -1
- package/assets/skills/wigolo/rules/synthesis.md +1 -1
- package/assets/skills/wigolo-fetch/SKILL.md +1 -1
- package/assets/skills/wigolo-find-similar/SKILL.md +2 -2
- package/assets/skills/wigolo-search/SKILL.md +3 -3
- package/dist/cache/store.d.ts +9 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +30 -4
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +67 -13
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/health.js +1 -1
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/status.js +1 -1
- package/dist/cli/status.js.map +1 -1
- package/dist/cli/tui/hooks/useInstall.js +2 -2
- package/dist/cli/tui/hooks/useInstall.js.map +1 -1
- package/dist/cli/tui/hooks/useVerify.js +3 -3
- package/dist/cli/tui/hooks/useVerify.js.map +1 -1
- package/dist/cli/tui/status-format.d.ts +1 -1
- package/dist/cli/tui/status-format.d.ts.map +1 -1
- package/dist/cli/tui/status-format.js +5 -5
- package/dist/cli/tui/status-format.js.map +1 -1
- package/dist/cli/tui/status-python.d.ts +1 -1
- package/dist/cli/tui/status-python.d.ts.map +1 -1
- package/dist/cli/tui/status-python.js +17 -1
- package/dist/cli/tui/status-python.js.map +1 -1
- package/dist/cli/tui/verify-suggestions.d.ts +1 -1
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -1
- package/dist/cli/tui/verify-suggestions.js +5 -5
- package/dist/cli/tui/verify-suggestions.js.map +1 -1
- package/dist/cli/tui/verify.d.ts +2 -2
- package/dist/cli/tui/verify.d.ts.map +1 -1
- package/dist/cli/tui/verify.js +34 -8
- package/dist/cli/tui/verify.js.map +1 -1
- package/dist/cli/uninstall.js +2 -2
- package/dist/cli/uninstall.js.map +1 -1
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +29 -25
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +6 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -2
- package/dist/config.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +47 -1
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +49 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +7 -3
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/jsonld.js +1 -1
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +28 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm/anthropic.d.ts +3 -0
- package/dist/extraction/llm/anthropic.d.ts.map +1 -0
- package/dist/extraction/llm/anthropic.js +33 -0
- package/dist/extraction/llm/anthropic.js.map +1 -0
- package/dist/extraction/llm/cache.d.ts +5 -0
- package/dist/extraction/llm/cache.d.ts.map +1 -0
- package/dist/extraction/llm/cache.js +35 -0
- package/dist/extraction/llm/cache.js.map +1 -0
- package/dist/extraction/llm/gemini.d.ts +3 -0
- package/dist/extraction/llm/gemini.d.ts.map +1 -0
- package/dist/extraction/llm/gemini.js +35 -0
- package/dist/extraction/llm/gemini.js.map +1 -0
- package/dist/extraction/llm/groq.d.ts +3 -0
- package/dist/extraction/llm/groq.d.ts.map +1 -0
- package/dist/extraction/llm/groq.js +63 -0
- package/dist/extraction/llm/groq.js.map +1 -0
- package/dist/extraction/llm/hash.d.ts +3 -0
- package/dist/extraction/llm/hash.d.ts.map +1 -0
- package/dist/extraction/llm/hash.js +22 -0
- package/dist/extraction/llm/hash.js.map +1 -0
- package/dist/extraction/llm/openai.d.ts +3 -0
- package/dist/extraction/llm/openai.d.ts.map +1 -0
- package/dist/extraction/llm/openai.js +38 -0
- package/dist/extraction/llm/openai.js.map +1 -0
- package/dist/extraction/llm/select.d.ts +5 -0
- package/dist/extraction/llm/select.d.ts.map +1 -0
- package/dist/extraction/llm/select.js +27 -0
- package/dist/extraction/llm/select.js.map +1 -0
- package/dist/extraction/llm/types.d.ts +24 -0
- package/dist/extraction/llm/types.d.ts.map +1 -0
- package/dist/extraction/llm/types.js +2 -0
- package/dist/extraction/llm/types.js.map +1 -0
- package/dist/extraction/llm/validate.d.ts +6 -0
- package/dist/extraction/llm/validate.d.ts.map +1 -0
- package/dist/extraction/llm/validate.js +63 -0
- package/dist/extraction/llm/validate.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +129 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown.d.ts +9 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +52 -3
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +17 -5
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +2 -3
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +81 -11
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +2 -3
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +4 -5
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +2 -3
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +3 -4
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +203 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/fetch/router.d.ts +2 -1
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +19 -1
- package/dist/fetch/router.js.map +1 -1
- package/dist/instructions.d.ts +8 -8
- package/dist/instructions.d.ts.map +1 -1
- package/dist/instructions.js +48 -41
- package/dist/instructions.js.map +1 -1
- package/dist/logger.d.ts +1 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/research/brief.js +1 -1
- package/dist/research/brief.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +260 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/highlights.d.ts +11 -2
- package/dist/search/highlights.d.ts.map +1 -1
- package/dist/search/highlights.js +131 -48
- package/dist/search/highlights.js.map +1 -1
- package/dist/search/multi-query.d.ts +1 -0
- package/dist/search/multi-query.d.ts.map +1 -1
- package/dist/search/multi-query.js +13 -0
- package/dist/search/multi-query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -2
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +16 -44
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/download.d.ts +9 -0
- package/dist/search/reranker/download.d.ts.map +1 -0
- package/dist/search/reranker/download.js +77 -0
- package/dist/search/reranker/download.js.map +1 -0
- package/dist/search/reranker/models.d.ts +14 -0
- package/dist/search/reranker/models.d.ts.map +1 -0
- package/dist/search/reranker/models.js +37 -0
- package/dist/search/reranker/models.js.map +1 -0
- package/dist/search/reranker/onnx.d.ts +13 -0
- package/dist/search/reranker/onnx.d.ts.map +1 -0
- package/dist/search/reranker/onnx.js +70 -0
- package/dist/search/reranker/onnx.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +12 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +26 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/tokenizer.d.ts +30 -0
- package/dist/search/reranker/tokenizer.d.ts.map +1 -0
- package/dist/search/reranker/tokenizer.js +49 -0
- package/dist/search/reranker/tokenizer.js.map +1 -0
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +38 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +4 -0
- package/dist/search/truncate.d.ts.map +1 -1
- package/dist/search/truncate.js +13 -0
- package/dist/search/truncate.js.map +1 -1
- package/dist/server/backend-status.js +2 -2
- package/dist/server/backend-status.js.map +1 -1
- package/dist/server/tool-schemas.d.ts +503 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +425 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +14 -339
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts.map +1 -1
- package/dist/tools/agent.js +36 -0
- package/dist/tools/agent.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +37 -2
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +19 -3
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +44 -7
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts.map +1 -1
- package/dist/tools/find-similar.js +32 -1
- package/dist/tools/find-similar.js.map +1 -1
- package/dist/tools/research.d.ts.map +1 -1
- package/dist/tools/research.js +34 -1
- package/dist/tools/research.js.map +1 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +98 -54
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +65 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +13 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +10 -4
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -64
- package/dist/search/flashrank.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"highlights.js","sourceRoot":"","sources":["../../src/search/highlights.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"highlights.js","sourceRoot":"","sources":["../../src/search/highlights.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEhF,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAwBlC,SAAS,UAAU,CAAC,OAAe;IACjC,IAAI,OAAO,CAAC,MAAM,GAAG,kBAAkB;QAAE,OAAO,KAAK,CAAC;IACtD,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,KAAK,CAAC;IACpE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,yEAAyE;AACzE,2DAA2D;AAC3D,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,CAAC;IACzB,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,MAAM,EAAE,GAAG,QAAQ,CAAC;IACpB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,CAAyB,CAAC;IAC9B,MAAM,QAAQ,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAE,EAAE;QACpD,wEAAwE;QACxE,MAAM,GAAG,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,OAAO,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClE,IAAI,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,OAAO,QAAQ,GAAG,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;YAAE,QAAQ,EAAE,CAAC;QACtE,IAAI,QAAQ,IAAI,OAAO;YAAE,OAAO;QAChC,MAAM,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAC;QACxC,MAAM,UAAU,GAAG,QAAQ,GAAG,QAAQ,CAAC;QACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,OAAO;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAClG,MAAM,OAAO,GAAG,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;IACvD,CAAC,CAAC;IACF,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9B,UAAU,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrC,CAAC;IACD,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,yEAAyE;AACzE,SAAS,uBAAuB,CAAC,QAAgB;IAC/C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACxD,CAAC;AAMD,0EAA0E;AAC1E,2EAA2E;AAC3E,6DAA6D;AAC7D,MAAM,UAAU,kBAAkB,CAChC,QAAgB,EAChB,QAAmB;IAEnB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAChG,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,IAAI,OAAO,GAAkB,IAAI,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS;gBAAE,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC;;gBAC5C,MAAM;QACb,CAAC;QACD,OAAO,EAAE,GAAG,CAAC,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,4EAA4E;AAC5E,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,OAA2B,EAC3B,gBAAwB,sBAAsB;IAE9C,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,UAAU,GAAuB,EAAE,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,SAAS,CAAC,IAAI,CAAC;YACb,KAAK,EAAE,CAAC,GAAG,CAAC;YACZ,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,CAAC,CAAC,gBAAgB,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,CAAC,GAAG,CAAC;gBAClB,SAAS,EAAE,CAAC,CAAC,GAAG;gBAChB,WAAW,EAAE,CAAC,CAAC,KAAK;gBACpB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,cAAc,EAAE,CAAC,CAAC,cAAc;aACjC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,UAAU,EAAE,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC;YACtD,SAAS;YACT,aAAa,EAAE,KAAK;SACrB,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,IAAI,GAAG,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,KAAK,EACL,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EACzC,EAAE,OAAO,EAAE,GAAG,CAAC,aAAa,EAAE,CAC/B,CAAC;YACF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAY,CAAC,CAAC,EAAE,EAAE;oBAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBACjC,OAAO;wBACL,IAAI,EAAE,IAAI,CAAC,IAAI;wBACf,YAAY,EAAE,IAAI,CAAC,WAAW;wBAC9B,eAAe,EAAE,CAAC,CAAC,KAAK;wBACxB,UAAU,EAAE,IAAI,CAAC,SAAS;wBAC1B,YAAY,EAAE,IAAI,CAAC,WAAW;wBAC9B,eAAe,EAAE,IAAI,CAAC,cAAc;wBACpC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBAC1D,CAAC;gBACJ,CAAC,CAAC,CAAC;gBACH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;YACxD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,kBAAkB,CAAC,OAAO,EAAE,aAAa,CAAC,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AACrG,CAAC;AAED,uFAAuF;AACvF,6EAA6E;AAC7E,6CAA6C;AAC7C,MAAM,UAAU,kBAAkB,CAChC,OAA2B,EAC3B,aAAqB;IAErB,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;QACtE,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,MAAM,GAAG,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YACzD,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI;gBACJ,YAAY,EAAE,CAAC,GAAG,CAAC;gBACnB,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,UAAU,EAAE,CAAC,CAAC,GAAG;gBACjB,YAAY,EAAE,CAAC,CAAC,KAAK;gBACrB,eAAe,EAAE,SAAS,CAAC,cAAc;gBACzC,WAAW,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE;aACpF,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,YAAY,EAAE,CAAC,GAAG,CAAC;YACnB,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,UAAU,EAAE,CAAC,CAAC,GAAG;YACjB,YAAY,EAAE,CAAC,CAAC,KAAK;YACrB,eAAe,EAAE,IAAI;YACrB,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE;SAC5C,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -19,4 +19,5 @@ export interface FanOutResult {
|
|
|
19
19
|
export declare function fanOutSearch(queries: string[], engines: SearchEngine[], options: FanOutOptions): Promise<FanOutResult>;
|
|
20
20
|
export declare function synthesizeIntent(queries: string[]): string;
|
|
21
21
|
export declare function mergeWithRRF(rankedLists: MergedSearchResult[][]): MergedSearchResult[];
|
|
22
|
+
export declare function expandQueryHeuristic(query: string): string[];
|
|
22
23
|
//# sourceMappingURL=multi-query.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query.d.ts","sourceRoot":"","sources":["../../src/search/multi-query.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AACtF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AASrD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CA2B5D;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;CACvE;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAsEvB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAO1D;AAED,wBAAgB,YAAY,CAAC,WAAW,EAAE,kBAAkB,EAAE,EAAE,GAAG,kBAAkB,EAAE,CAmDtF"}
|
|
1
|
+
{"version":3,"file":"multi-query.d.ts","sourceRoot":"","sources":["../../src/search/multi-query.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AACtF,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AASrD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CA2B5D;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;CACvE;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CAsEvB;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAO1D;AAED,wBAAgB,YAAY,CAAC,WAAW,EAAE,kBAAkB,EAAE,EAAE,GAAG,kBAAkB,EAAE,CAmDtF;AAID,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAS5D"}
|
|
@@ -154,4 +154,17 @@ export function mergeWithRRF(rankedLists) {
|
|
|
154
154
|
return [];
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
|
+
const DEEP_SUFFIXES = ['guide', 'tutorial', 'examples', 'best practices'];
|
|
158
|
+
export function expandQueryHeuristic(query) {
|
|
159
|
+
const trimmed = query.trim();
|
|
160
|
+
const variants = [trimmed];
|
|
161
|
+
for (const suffix of DEEP_SUFFIXES) {
|
|
162
|
+
if (variants.length >= 5)
|
|
163
|
+
break;
|
|
164
|
+
const candidate = `${trimmed} ${suffix}`;
|
|
165
|
+
if (!variants.includes(candidate))
|
|
166
|
+
variants.push(candidate);
|
|
167
|
+
}
|
|
168
|
+
return variants;
|
|
169
|
+
}
|
|
157
170
|
//# sourceMappingURL=multi-query.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"multi-query.js","sourceRoot":"","sources":["../../src/search/multi-query.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACZ,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBACpD,QAAQ,EAAE,UAAU,CAAC,MAAM;gBAC3B,GAAG,EAAE,MAAM,CAAC,aAAa;aAC1B,CAAC,CAAC;YACH,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAmBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAiB,EACjB,OAAuB,EACvB,OAAsB;IAEtB,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACtD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,qBAAqB,CAAC;QAEjD,MAAM,kBAAkB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAChG,MAAM,eAAe,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnD,MAAM,aAAa,GAAwB;YACzC,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,eAAe;YAChD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC;QAEF,MAAM,KAAK,GAAmD,EAAE,CAAC;QACjE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;gBACrD,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;oBAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;wBACxB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;wBACnB,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7D,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE;wBAC3C,MAAM,EAAE,MAAM,CAAC,IAAI;wBACnB,KAAK;wBACL,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;oBACH,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,KAAK,MAAM,GAAG,EAAE,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;YAC7B,MAAM;SACP,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;YAC7B,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,iBAAiB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,WAAmC;IAC9D,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4D,CAAC;QAE3F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,IAAI,gBAAwB,CAAC;gBAC7B,IAAI,CAAC;oBACH,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,CAAC;gBAAC,MAAM,CAAC;oBACP,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC;gBAC9B,CAAC;gBAED,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBACrD,SAAS,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,GAAG,eAAe,CAAC,CAAC;gBAE3D,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBACtD,IAAI,CAAC,QAAQ,IAAI,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC1C,cAAc,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,KAAK,GAAG,QAAQ;gBAAE,QAAQ,GAAG,KAAK,CAAC;QACzC,CAAC;QAED,MAAM,MAAM,GAAyB,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5D,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,UAAU,CAAC,MAAM;gBACpB,eAAe,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;aACrD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"multi-query.js","sourceRoot":"","sources":["../../src/search/multi-query.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACZ,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;YAC7C,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;gBACpD,QAAQ,EAAE,UAAU,CAAC,MAAM;gBAC3B,GAAG,EAAE,MAAM,CAAC,aAAa;aAC1B,CAAC,CAAC;YACH,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAmBD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAiB,EACjB,OAAuB,EACvB,OAAsB;IAEtB,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACtD,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,qBAAqB,CAAC;QAEjD,MAAM,kBAAkB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAChG,MAAM,eAAe,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnD,MAAM,aAAa,GAAwB;YACzC,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,eAAe;YAChD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC;QAEF,MAAM,KAAK,GAAmD,EAAE,CAAC;QACjE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;gBACrD,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;oBAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;wBACxB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;wBACnB,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC/B,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7D,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE;wBAC3C,MAAM,EAAE,MAAM,CAAC,IAAI;wBACnB,KAAK;wBACL,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;oBACH,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,KAAK,MAAM,GAAG,EAAE,CAAC,CAAC;gBAClD,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;YAC7B,MAAM;SACP,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO;YACL,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;YAC7B,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,iBAAiB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,WAAmC;IAC9D,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,cAAc,GAAG,IAAI,GAAG,EAA4D,CAAC;QAE3F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,IAAI,gBAAwB,CAAC;gBAC7B,IAAI,CAAC;oBACH,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,CAAC;gBAAC,MAAM,CAAC;oBACP,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC;gBAC9B,CAAC;gBAED,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBACrD,SAAS,CAAC,GAAG,CAAC,gBAAgB,EAAE,OAAO,GAAG,eAAe,CAAC,CAAC;gBAE3D,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;gBACtD,IAAI,CAAC,QAAQ,IAAI,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC1C,cAAc,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,KAAK,GAAG,QAAQ;gBAAE,QAAQ,GAAG,KAAK,CAAC;QACzC,CAAC;QAED,MAAM,MAAM,GAAyB,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,gBAAgB,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5D,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,gBAAgB,CAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC;gBACV,GAAG,UAAU,CAAC,MAAM;gBACpB,eAAe,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;aACrD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,gBAAgB,CAAU,CAAC;AAEnF,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAa,CAAC,OAAO,CAAC,CAAC;IACrC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC;YAAE,MAAM;QAChC,MAAM,SAAS,GAAG,GAAG,OAAO,IAAI,MAAM,EAAE,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/search/rerank.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { MergedSearchResult } from './dedup.js';
|
|
2
|
-
export declare function rerankResults(query: string, results: MergedSearchResult[]
|
|
3
|
-
|
|
2
|
+
export declare function rerankResults(query: string, results: MergedSearchResult[], opts?: {
|
|
3
|
+
skip?: boolean;
|
|
4
|
+
}): Promise<MergedSearchResult[]>;
|
|
4
5
|
//# sourceMappingURL=rerank.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"rerank.d.ts","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAQrD,wBAAsB,aAAa,CACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAAE,EAC7B,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAuB/B"}
|
package/dist/search/rerank.js
CHANGED
|
@@ -1,62 +1,34 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { onnxRerank } from './reranker/onnx.js';
|
|
2
|
+
import { applyRecencyBoost } from './reranker/recency-boost.js';
|
|
2
3
|
import { getConfig } from '../config.js';
|
|
3
4
|
import { createLogger } from '../logger.js';
|
|
4
5
|
const log = createLogger('search');
|
|
5
|
-
export async function rerankResults(query, results) {
|
|
6
|
+
export async function rerankResults(query, results, opts = {}) {
|
|
7
|
+
if (opts.skip)
|
|
8
|
+
return results;
|
|
6
9
|
const config = getConfig();
|
|
7
10
|
if (results.length === 0)
|
|
8
11
|
return results;
|
|
9
|
-
if (config.reranker === '
|
|
10
|
-
|
|
11
|
-
const passages = results.map((r
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const reordered = ranked.map((r) => ({
|
|
18
|
-
...results[r.index],
|
|
19
|
-
relevance_score: r.score,
|
|
20
|
-
}));
|
|
21
|
-
const boosted = applyRecencyBoost(reordered);
|
|
22
|
-
boosted.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
23
|
-
return applyThreshold(boosted, config.relevanceThreshold);
|
|
24
|
-
}
|
|
25
|
-
log.debug('FlashRank returned null, using passthrough');
|
|
12
|
+
if (config.reranker === 'onnx') {
|
|
13
|
+
try {
|
|
14
|
+
const passages = results.map((r) => ({ text: `${r.title}\n${r.snippet}` }));
|
|
15
|
+
const ranked = await onnxRerank(query, passages, { modelId: config.rerankerModel });
|
|
16
|
+
const reordered = ranked.map((s) => ({ ...results[s.index], relevance_score: s.score }));
|
|
17
|
+
const boosted = applyRecencyBoost(query, reordered);
|
|
18
|
+
boosted.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
19
|
+
return applyThreshold(boosted, config.relevanceThreshold);
|
|
26
20
|
}
|
|
27
|
-
|
|
28
|
-
log.warn('
|
|
21
|
+
catch (err) {
|
|
22
|
+
log.warn('ONNX rerank failed, falling back to passthrough', { error: String(err) });
|
|
29
23
|
}
|
|
30
24
|
}
|
|
31
25
|
else if (config.reranker !== 'none') {
|
|
32
26
|
log.warn('Unknown reranker configured, passing through', { reranker: config.reranker });
|
|
33
27
|
}
|
|
34
|
-
|
|
35
|
-
const boosted = applyRecencyBoost(results);
|
|
28
|
+
const boosted = applyRecencyBoost(query, results);
|
|
36
29
|
boosted.sort((a, b) => b.relevance_score - a.relevance_score);
|
|
37
30
|
return applyThreshold(boosted, config.relevanceThreshold);
|
|
38
31
|
}
|
|
39
|
-
export function applyRecencyBoost(results) {
|
|
40
|
-
const now = Date.now();
|
|
41
|
-
return results.map(r => {
|
|
42
|
-
if (!r.published_date)
|
|
43
|
-
return r;
|
|
44
|
-
const ts = new Date(r.published_date).getTime();
|
|
45
|
-
if (isNaN(ts))
|
|
46
|
-
return r;
|
|
47
|
-
const ageDays = (now - ts) / (1000 * 60 * 60 * 24);
|
|
48
|
-
let boost = 1.0;
|
|
49
|
-
if (ageDays < 7)
|
|
50
|
-
boost = 1.2;
|
|
51
|
-
else if (ageDays < 30)
|
|
52
|
-
boost = 1.1;
|
|
53
|
-
else if (ageDays < 90)
|
|
54
|
-
boost = 1.05;
|
|
55
|
-
if (boost === 1.0)
|
|
56
|
-
return r;
|
|
57
|
-
return { ...r, relevance_score: r.relevance_score * boost };
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
32
|
function applyThreshold(results, threshold) {
|
|
61
33
|
if (!threshold || threshold <= 0)
|
|
62
34
|
return results;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rerank.js","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"rerank.js","sourceRoot":"","sources":["../../src/search/rerank.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAa,EACb,OAA6B,EAC7B,OAA2B,EAAE;IAE7B,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,OAAO,CAAC;IAC9B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAEzC,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAC5E,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;YACpF,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACzF,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;YAC9D,OAAO,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAC5D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC;QACtC,GAAG,CAAC,IAAI,CAAC,8CAA8C,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;IAC9D,OAAO,cAAc,CAAC,OAAO,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CACrB,OAA6B,EAC7B,SAAiB;IAEjB,IAAI,CAAC,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IACjD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,SAAS,CAAC,CAAC;AAC/D,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { WarmupReporter } from '../../cli/tui/reporter.js';
|
|
2
|
+
export declare function _setFetchImpl(impl: typeof fetch): void;
|
|
3
|
+
export interface DownloadPaths {
|
|
4
|
+
modelPath: string;
|
|
5
|
+
tokenizerPath: string;
|
|
6
|
+
configPath: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function downloadModelAssets(modelId: string, dataDir: string, reporter?: WarmupReporter): Promise<DownloadPaths>;
|
|
9
|
+
//# sourceMappingURL=download.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/download.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAKhE,wBAAgB,aAAa,CAAC,IAAI,EAAE,OAAO,KAAK,GAAG,IAAI,CAEtD;AAQD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,cAAc,GACxB,OAAO,CAAC,aAAa,CAAC,CA4BxB"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { createWriteStream, mkdirSync, readFileSync, existsSync, unlinkSync } from 'node:fs';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { join, dirname } from 'node:path';
|
|
4
|
+
import { getModel, resolveModelId } from './models.js';
|
|
5
|
+
import { createLogger } from '../../logger.js';
|
|
6
|
+
const log = createLogger('reranker');
|
|
7
|
+
let fetchImpl = globalThis.fetch;
|
|
8
|
+
export function _setFetchImpl(impl) {
|
|
9
|
+
fetchImpl = impl;
|
|
10
|
+
}
|
|
11
|
+
const FILES = [
|
|
12
|
+
{ key: 'modelUrl', shaKey: 'modelSha256', filename: 'model_quantized.onnx', label: 'model' },
|
|
13
|
+
{ key: 'tokenizerUrl', shaKey: 'tokenizerSha256', filename: 'tokenizer.json', label: 'tokenizer' },
|
|
14
|
+
{ key: 'configUrl', shaKey: 'configSha256', filename: 'tokenizer_config.json', label: 'config' },
|
|
15
|
+
];
|
|
16
|
+
export async function downloadModelAssets(modelId, dataDir, reporter) {
|
|
17
|
+
const id = resolveModelId(modelId);
|
|
18
|
+
const m = getModel(id);
|
|
19
|
+
const targetDir = join(dataDir, 'models', id);
|
|
20
|
+
mkdirSync(targetDir, { recursive: true });
|
|
21
|
+
for (const f of FILES) {
|
|
22
|
+
const dest = join(targetDir, f.filename);
|
|
23
|
+
const expectedSha = m[f.shaKey];
|
|
24
|
+
if (existsSync(dest)) {
|
|
25
|
+
const actual = sha256Of(dest);
|
|
26
|
+
if (actual === expectedSha) {
|
|
27
|
+
log.debug('cached asset valid', { id, file: f.filename });
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
log.warn('cached asset corrupt — re-downloading', { id, file: f.filename, expected: expectedSha, actual });
|
|
31
|
+
unlinkSync(dest);
|
|
32
|
+
}
|
|
33
|
+
await downloadOne(m[f.key], dest, expectedSha, reporter, f.label);
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
modelPath: join(targetDir, 'model_quantized.onnx'),
|
|
37
|
+
tokenizerPath: join(targetDir, 'tokenizer.json'),
|
|
38
|
+
configPath: join(targetDir, 'tokenizer_config.json'),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
async function downloadOne(url, dest, expectedSha, reporter, label) {
|
|
42
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
43
|
+
const resp = await fetchImpl(url);
|
|
44
|
+
if (!resp.ok || !resp.body)
|
|
45
|
+
throw new Error(`HTTP ${resp.status} fetching ${url}`);
|
|
46
|
+
const headers = resp.headers;
|
|
47
|
+
const totalRaw = headers.get?.('content-length') ?? '0';
|
|
48
|
+
const total = Number(totalRaw);
|
|
49
|
+
let downloaded = 0;
|
|
50
|
+
const ws = createWriteStream(dest);
|
|
51
|
+
const reader = resp.body.getReader();
|
|
52
|
+
try {
|
|
53
|
+
while (true) {
|
|
54
|
+
const { done, value } = await reader.read();
|
|
55
|
+
if (done || !value)
|
|
56
|
+
break;
|
|
57
|
+
if (!ws.write(value))
|
|
58
|
+
await new Promise((r) => ws.once('drain', r));
|
|
59
|
+
downloaded += value.byteLength;
|
|
60
|
+
if (total > 0 && reporter)
|
|
61
|
+
reporter.progress('reranker', downloaded / total);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
ws.end();
|
|
66
|
+
await new Promise((r) => ws.once('finish', () => r()));
|
|
67
|
+
}
|
|
68
|
+
const actual = sha256Of(dest);
|
|
69
|
+
if (actual !== expectedSha) {
|
|
70
|
+
unlinkSync(dest);
|
|
71
|
+
throw new Error(`SHA-256 mismatch for ${label} at ${url}: expected ${expectedSha}, got ${actual}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function sha256Of(path) {
|
|
75
|
+
return createHash('sha256').update(readFileSync(path)).digest('hex');
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=download.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../src/search/reranker/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG/C,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAErC,IAAI,SAAS,GAAiB,UAAU,CAAC,KAAK,CAAC;AAC/C,MAAM,UAAU,aAAa,CAAC,IAAkB;IAC9C,SAAS,GAAG,IAAI,CAAC;AACnB,CAAC;AAED,MAAM,KAAK,GAAG;IACZ,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE;IAC5F,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,iBAAiB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,KAAK,EAAE,WAAW,EAAE;IAClG,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,uBAAuB,EAAE,KAAK,EAAE,QAAQ,EAAE;CACxF,CAAC;AAQX,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAe,EACf,OAAe,EACf,QAAyB;IAEzB,MAAM,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE,CAA+C,CAAC;IACrE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC9C,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAW,CAAC;QAE1C,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;gBAC3B,GAAG,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,SAAS;YACX,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3G,UAAU,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QAED,MAAM,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAW,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO;QACL,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,sBAAsB,CAAC;QAClD,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC;QAChD,UAAU,EAAE,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAW,EACX,IAAY,EACZ,WAAmB,EACnB,QAAoC,EACpC,KAAa;IAEb,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,IAAI,CAAC,OAA4D,CAAC;IAClF,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,EAAE,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,MAAM,GAAI,IAAI,CAAC,IAAuG,CAAC,SAAS,EAAE,CAAC;IACzI,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM;YAC1B,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YAC1E,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;YAC/B,IAAI,KAAK,GAAG,CAAC,IAAI,QAAQ;gBAAE,QAAQ,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,KAAK,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,UAAU,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,OAAO,GAAG,cAAc,WAAW,SAAS,MAAM,EAAE,CAAC,CAAC;IACrG,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface ModelEntry {
|
|
2
|
+
id: string;
|
|
3
|
+
modelUrl: string;
|
|
4
|
+
modelSha256: string;
|
|
5
|
+
tokenizerUrl: string;
|
|
6
|
+
tokenizerSha256: string;
|
|
7
|
+
configUrl: string;
|
|
8
|
+
configSha256: string;
|
|
9
|
+
approxBytes: number;
|
|
10
|
+
}
|
|
11
|
+
export declare const MODELS: Record<string, ModelEntry>;
|
|
12
|
+
export declare function resolveModelId(input: string): string;
|
|
13
|
+
export declare function getModel(id: string): ModelEntry;
|
|
14
|
+
//# sourceMappingURL=models.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/models.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CA2B7C,CAAC;AAMF,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,UAAU,CAO/C"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export const MODELS = {
|
|
2
|
+
'bge-reranker-v2-m3': {
|
|
3
|
+
id: 'bge-reranker-v2-m3',
|
|
4
|
+
modelUrl: 'https://huggingface.co/onnx-community/bge-reranker-v2-m3-ONNX/resolve/main/onnx/model_quantized.onnx',
|
|
5
|
+
modelSha256: '912fc1215c2dbff6499700534bd8d31253af01573861abbfc43afd1fab6cce5d',
|
|
6
|
+
tokenizerUrl: 'https://huggingface.co/onnx-community/bge-reranker-v2-m3-ONNX/resolve/main/tokenizer.json',
|
|
7
|
+
tokenizerSha256: '8bf8afbfd11306bd872018c53bfdf2e160a56f8edbcf49933324404791c148d3',
|
|
8
|
+
configUrl: 'https://huggingface.co/onnx-community/bge-reranker-v2-m3-ONNX/resolve/main/tokenizer_config.json',
|
|
9
|
+
configSha256: 'b87c8703482b0300d3da30e201519aa641f6a450f5eb5bf1e624afbf70c74d80',
|
|
10
|
+
approxBytes: 570_727_094,
|
|
11
|
+
},
|
|
12
|
+
'ms-marco-MiniLM-L-12-v2': {
|
|
13
|
+
id: 'ms-marco-MiniLM-L-12-v2',
|
|
14
|
+
modelUrl: 'https://huggingface.co/Xenova/ms-marco-MiniLM-L-12-v2/resolve/main/onnx/model_quantized.onnx',
|
|
15
|
+
modelSha256: 'c5551b3e446396364913c5ad79e9c8411a76d26523b7d87232052ae6c0d0c7fd',
|
|
16
|
+
tokenizerUrl: 'https://huggingface.co/Xenova/ms-marco-MiniLM-L-12-v2/resolve/main/tokenizer.json',
|
|
17
|
+
tokenizerSha256: 'd241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66',
|
|
18
|
+
configUrl: 'https://huggingface.co/Xenova/ms-marco-MiniLM-L-12-v2/resolve/main/tokenizer_config.json',
|
|
19
|
+
configSha256: '0b29c7bfc889e53b36d9dd3e686dd4300f6525110eaa98c76a5dafceb2029f53',
|
|
20
|
+
approxBytes: 34_201_341,
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
const ALIASES = {
|
|
24
|
+
'minilm-l12': 'ms-marco-MiniLM-L-12-v2',
|
|
25
|
+
};
|
|
26
|
+
export function resolveModelId(input) {
|
|
27
|
+
return ALIASES[input] ?? input;
|
|
28
|
+
}
|
|
29
|
+
export function getModel(id) {
|
|
30
|
+
const resolved = resolveModelId(id);
|
|
31
|
+
const entry = MODELS[resolved];
|
|
32
|
+
if (!entry) {
|
|
33
|
+
throw new Error(`Unknown reranker model: ${id} (resolved to ${resolved})`);
|
|
34
|
+
}
|
|
35
|
+
return entry;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=models.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/search/reranker/models.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,MAAM,GAA+B;IAChD,oBAAoB,EAAE;QACpB,EAAE,EAAE,oBAAoB;QACxB,QAAQ,EACN,sGAAsG;QACxG,WAAW,EAAE,kEAAkE;QAC/E,YAAY,EACV,2FAA2F;QAC7F,eAAe,EAAE,kEAAkE;QACnF,SAAS,EACP,kGAAkG;QACpG,YAAY,EAAE,kEAAkE;QAChF,WAAW,EAAE,WAAW;KACzB;IACD,yBAAyB,EAAE;QACzB,EAAE,EAAE,yBAAyB;QAC7B,QAAQ,EACN,8FAA8F;QAChG,WAAW,EAAE,kEAAkE;QAC/E,YAAY,EACV,mFAAmF;QACrF,eAAe,EAAE,kEAAkE;QACnF,SAAS,EACP,0FAA0F;QAC5F,YAAY,EAAE,kEAAkE;QAChF,WAAW,EAAE,UAAU;KACxB;CACF,CAAC;AAEF,MAAM,OAAO,GAA2B;IACtC,YAAY,EAAE,yBAAyB;CACxC,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,EAAU;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,EAAE,iBAAiB,QAAQ,GAAG,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface RerankDoc {
|
|
2
|
+
text: string;
|
|
3
|
+
}
|
|
4
|
+
export interface RerankScore {
|
|
5
|
+
index: number;
|
|
6
|
+
score: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function _resetOnnxSessionCache(): void;
|
|
9
|
+
export declare function onnxRerank(query: string, docs: RerankDoc[], opts?: {
|
|
10
|
+
modelId?: string;
|
|
11
|
+
maxLength?: number;
|
|
12
|
+
}): Promise<RerankScore[]>;
|
|
13
|
+
//# sourceMappingURL=onnx.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/onnx.ts"],"names":[],"mappings":"AASA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;CACd;AACD,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAID,wBAAgB,sBAAsB,IAAI,IAAI,CAE7C;AA0BD,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAClD,OAAO,CAAC,WAAW,EAAE,CAAC,CAwCxB"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import * as ort from 'onnxruntime-node';
|
|
2
|
+
import { downloadModelAssets } from './download.js';
|
|
3
|
+
import { loadTokenizer, tokenizePair } from './tokenizer.js';
|
|
4
|
+
import { getConfig } from '../../config.js';
|
|
5
|
+
import { createLogger } from '../../logger.js';
|
|
6
|
+
import { resolveModelId } from './models.js';
|
|
7
|
+
const log = createLogger('reranker');
|
|
8
|
+
const sessionCache = new Map();
|
|
9
|
+
export function _resetOnnxSessionCache() {
|
|
10
|
+
sessionCache.clear();
|
|
11
|
+
}
|
|
12
|
+
async function getSession(modelId, modelPath) {
|
|
13
|
+
const id = resolveModelId(modelId);
|
|
14
|
+
const existing = sessionCache.get(id);
|
|
15
|
+
if (existing)
|
|
16
|
+
return existing;
|
|
17
|
+
const p = (async () => {
|
|
18
|
+
log.debug('creating ONNX session', { id, modelPath });
|
|
19
|
+
return ort.InferenceSession.create(modelPath, {
|
|
20
|
+
executionProviders: ['cpu'],
|
|
21
|
+
graphOptimizationLevel: 'all',
|
|
22
|
+
});
|
|
23
|
+
})();
|
|
24
|
+
sessionCache.set(id, p);
|
|
25
|
+
try {
|
|
26
|
+
return await p;
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
sessionCache.delete(id);
|
|
30
|
+
throw err;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function sigmoid(x) {
|
|
34
|
+
return 1 / (1 + Math.exp(-x));
|
|
35
|
+
}
|
|
36
|
+
export async function onnxRerank(query, docs, opts = {}) {
|
|
37
|
+
if (docs.length === 0)
|
|
38
|
+
return [];
|
|
39
|
+
const cfg = getConfig();
|
|
40
|
+
const modelId = opts.modelId ?? cfg.rerankerModel ?? 'bge-reranker-v2-m3';
|
|
41
|
+
const maxLength = opts.maxLength ?? 512;
|
|
42
|
+
// Order matters: download assets first (so tokenizer.json is on disk), then
|
|
43
|
+
// load the tokenizer via @xenova/transformers (which bundles its own ORT
|
|
44
|
+
// build and initializes shared native state on import), and only then
|
|
45
|
+
// create the onnxruntime-node session. Creating the ORT session before the
|
|
46
|
+
// xenova import overwrites our env handle and the next session.run()
|
|
47
|
+
// segfaults (SIGBUS) on macOS.
|
|
48
|
+
const { modelPath } = await downloadModelAssets(modelId, cfg.dataDir);
|
|
49
|
+
const tokenizer = await loadTokenizer(modelId, cfg.dataDir);
|
|
50
|
+
const session = await getSession(modelId, modelPath);
|
|
51
|
+
const scores = [];
|
|
52
|
+
for (let i = 0; i < docs.length; i++) {
|
|
53
|
+
const { input_ids, attention_mask, token_type_ids, length } = tokenizePair(tokenizer, query, docs[i].text, maxLength);
|
|
54
|
+
const dims = [1, length];
|
|
55
|
+
const feeds = {
|
|
56
|
+
input_ids: new ort.Tensor('int64', input_ids, dims),
|
|
57
|
+
attention_mask: new ort.Tensor('int64', attention_mask, dims),
|
|
58
|
+
};
|
|
59
|
+
if (session.inputNames.includes('token_type_ids')) {
|
|
60
|
+
feeds.token_type_ids = new ort.Tensor('int64', token_type_ids, dims);
|
|
61
|
+
}
|
|
62
|
+
const out = await session.run(feeds);
|
|
63
|
+
const logitsName = session.outputNames[0];
|
|
64
|
+
const logit = out[logitsName].data[0];
|
|
65
|
+
scores.push({ index: i, score: sigmoid(logit) });
|
|
66
|
+
}
|
|
67
|
+
scores.sort((a, b) => b.score - a.score);
|
|
68
|
+
return scores;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=onnx.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx.js","sourceRoot":"","sources":["../../../src/search/reranker/onnx.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAUrC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAyC,CAAC;AAEtE,MAAM,UAAU,sBAAsB;IACpC,YAAY,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,OAAe,EAAE,SAAiB;IAC1D,MAAM,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtC,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9B,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;QACpB,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QACtD,OAAO,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE;YAC5C,kBAAkB,EAAE,CAAC,KAAK,CAAC;YAC3B,sBAAsB,EAAE,KAAK;SAC9B,CAAC,CAAC;IACL,CAAC,CAAC,EAAE,CAAC;IACL,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,CAAC,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACxB,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAa,EACb,IAAiB,EACjB,OAAiD,EAAE;IAEnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;IACxB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,aAAa,IAAI,oBAAoB,CAAC;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IAExC,4EAA4E;IAC5E,yEAAyE;IACzE,sEAAsE;IACtE,2EAA2E;IAC3E,qEAAqE;IACrE,+BAA+B;IAC/B,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAErD,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,YAAY,CACxE,SAA+C,EAC/C,KAAK,EACL,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EACZ,SAAS,CACV,CAAC;QACF,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACzB,MAAM,KAAK,GAA+B;YACxC,SAAS,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC;YACnD,cAAc,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC;SAC9D,CAAC;QACF,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAClD,KAAK,CAAC,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAI,GAAG,CAAC,UAAU,CAAC,CAAC,IAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency-boost.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/recency-boost.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGtD,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAAE,EAC7B,GAAG,GAAE,IAAiB,GACrB,kBAAkB,EAAE,CAOtB"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { hasRecencyIntent, recencyFactor } from './recency.js';
|
|
2
|
+
export function applyRecencyBoost(query, results, now = new Date()) {
|
|
3
|
+
if (!hasRecencyIntent(query, now))
|
|
4
|
+
return results;
|
|
5
|
+
return results.map((r) => {
|
|
6
|
+
const factor = recencyFactor(r.published_date, now);
|
|
7
|
+
if (factor === 1.0)
|
|
8
|
+
return r;
|
|
9
|
+
return { ...r, relevance_score: r.relevance_score * factor };
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=recency-boost.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency-boost.js","sourceRoot":"","sources":["../../../src/search/reranker/recency-boost.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE/D,MAAM,UAAU,iBAAiB,CAC/B,KAAa,EACb,OAA6B,EAC7B,MAAY,IAAI,IAAI,EAAE;IAEtB,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QACpD,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,CAAC,CAAC;QAC7B,OAAO,EAAE,GAAG,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,GAAG,MAAM,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/recency.ts"],"names":[],"mappings":"AAEA,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,IAAiB,GAAG,OAAO,CAM/E;AAED,wBAAgB,aAAa,CAAC,aAAa,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,IAAiB,GAAG,MAAM,CAS/F"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
const RECENCY_TOKENS = /\b(recent|latest|new|just released|today|this week)\b/i;
|
|
2
|
+
export function hasRecencyIntent(query, now = new Date()) {
|
|
3
|
+
if (RECENCY_TOKENS.test(query))
|
|
4
|
+
return true;
|
|
5
|
+
const yearMatches = query.match(/\b(20\d{2})\b/g);
|
|
6
|
+
if (!yearMatches)
|
|
7
|
+
return false;
|
|
8
|
+
const currentYear = now.getUTCFullYear();
|
|
9
|
+
return yearMatches.some((y) => parseInt(y, 10) >= currentYear);
|
|
10
|
+
}
|
|
11
|
+
export function recencyFactor(publishedDate, now = new Date()) {
|
|
12
|
+
if (!publishedDate)
|
|
13
|
+
return 1.0;
|
|
14
|
+
const ts = new Date(publishedDate).getTime();
|
|
15
|
+
if (isNaN(ts))
|
|
16
|
+
return 1.0;
|
|
17
|
+
const ageDays = (now.getTime() - ts) / (1000 * 60 * 60 * 24);
|
|
18
|
+
if (ageDays < 7)
|
|
19
|
+
return 1.5;
|
|
20
|
+
if (ageDays < 30)
|
|
21
|
+
return 1.3;
|
|
22
|
+
if (ageDays < 90)
|
|
23
|
+
return 1.1;
|
|
24
|
+
return 1.0;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=recency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recency.js","sourceRoot":"","sources":["../../../src/search/reranker/recency.ts"],"names":[],"mappings":"AAAA,MAAM,cAAc,GAAG,wDAAwD,CAAC;AAEhF,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,MAAY,IAAI,IAAI,EAAE;IACpE,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAClD,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAC/B,MAAM,WAAW,GAAG,GAAG,CAAC,cAAc,EAAE,CAAC;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,WAAW,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,aAAiC,EAAE,MAAY,IAAI,IAAI,EAAE;IACrF,IAAI,CAAC,aAAa;QAAE,OAAO,GAAG,CAAC;IAC/B,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7C,IAAI,KAAK,CAAC,EAAE,CAAC;QAAE,OAAO,GAAG,CAAC;IAC1B,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC7D,IAAI,OAAO,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC;IAC7B,IAAI,OAAO,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC;IAC7B,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export type TokenizerPair = {
|
|
2
|
+
input_ids: BigInt64Array;
|
|
3
|
+
attention_mask: BigInt64Array;
|
|
4
|
+
token_type_ids: BigInt64Array;
|
|
5
|
+
length: number;
|
|
6
|
+
};
|
|
7
|
+
type EncodeOpts = {
|
|
8
|
+
text_pair: string;
|
|
9
|
+
max_length: number;
|
|
10
|
+
truncation: true;
|
|
11
|
+
padding: 'max_length';
|
|
12
|
+
return_tensor?: true;
|
|
13
|
+
};
|
|
14
|
+
type EncodedTensor = {
|
|
15
|
+
data: BigInt64Array;
|
|
16
|
+
dims: number[];
|
|
17
|
+
};
|
|
18
|
+
type EncodedBatch = {
|
|
19
|
+
input_ids: EncodedTensor;
|
|
20
|
+
attention_mask: EncodedTensor;
|
|
21
|
+
token_type_ids?: EncodedTensor;
|
|
22
|
+
};
|
|
23
|
+
type EncoderTokenizer = ((q: string, opts: EncodeOpts) => EncodedBatch) | {
|
|
24
|
+
encode: (q: string, opts: EncodeOpts) => EncodedBatch;
|
|
25
|
+
};
|
|
26
|
+
export declare function loadTokenizer(modelId: string, dataDir?: string): Promise<unknown>;
|
|
27
|
+
export declare function tokenizePair(tokenizer: EncoderTokenizer, query: string, doc: string, maxLength?: number): TokenizerPair;
|
|
28
|
+
export declare function _resetTokenizerCache(): void;
|
|
29
|
+
export {};
|
|
30
|
+
//# sourceMappingURL=tokenizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../../src/search/reranker/tokenizer.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,aAAa,CAAC;IACzB,cAAc,EAAE,aAAa,CAAC;IAC9B,cAAc,EAAE,aAAa,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,IAAI,CAAC;IACjB,OAAO,EAAE,YAAY,CAAC;IACtB,aAAa,CAAC,EAAE,IAAI,CAAC;CACtB,CAAC;AAEF,KAAK,aAAa,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAE7D,KAAK,YAAY,GAAG;IAClB,SAAS,EAAE,aAAa,CAAC;IACzB,cAAc,EAAE,aAAa,CAAC;IAC9B,cAAc,CAAC,EAAE,aAAa,CAAC;CAChC,CAAC;AAKF,KAAK,gBAAgB,GACjB,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,KAAK,YAAY,CAAC,GAC/C;IAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,KAAK,YAAY,CAAA;CAAE,CAAC;AAU9D,wBAAsB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAavF;AAED,wBAAgB,YAAY,CAC1B,SAAS,EAAE,gBAAgB,EAC3B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,SAAS,SAAM,GACd,aAAa,CAoBf;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C"}
|