okf-catalog 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/LICENSE +202 -0
- package/NOTICE +10 -0
- package/README.md +87 -0
- package/dist/bundle/cat-file.js +94 -0
- package/dist/bundle/cat-file.js.map +1 -0
- package/dist/bundle/contract.js +60 -0
- package/dist/bundle/contract.js.map +1 -0
- package/dist/bundle/frontmatter.js +106 -0
- package/dist/bundle/frontmatter.js.map +1 -0
- package/dist/bundle/git-tree.js +109 -0
- package/dist/bundle/git-tree.js.map +1 -0
- package/dist/bundle/index-file.js +90 -0
- package/dist/bundle/index-file.js.map +1 -0
- package/dist/bundle/links.js +57 -0
- package/dist/bundle/links.js.map +1 -0
- package/dist/bundle/load.js +363 -0
- package/dist/bundle/load.js.map +1 -0
- package/dist/bundle/manifest.js +95 -0
- package/dist/bundle/manifest.js.map +1 -0
- package/dist/bundle/markdown.js +210 -0
- package/dist/bundle/markdown.js.map +1 -0
- package/dist/bundle/model.js +7 -0
- package/dist/bundle/model.js.map +1 -0
- package/dist/bundle/page.js +374 -0
- package/dist/bundle/page.js.map +1 -0
- package/dist/bundle/paths.js +36 -0
- package/dist/bundle/paths.js.map +1 -0
- package/dist/bundle/reserved.js +56 -0
- package/dist/bundle/reserved.js.map +1 -0
- package/dist/bundle/timestamp.js +49 -0
- package/dist/bundle/timestamp.js.map +1 -0
- package/dist/catalog/model.js +32 -0
- package/dist/catalog/model.js.map +1 -0
- package/dist/catalog/nearest.js +56 -0
- package/dist/catalog/nearest.js.map +1 -0
- package/dist/catalog/outputs.js +438 -0
- package/dist/catalog/outputs.js.map +1 -0
- package/dist/catalog/provenance.js +37 -0
- package/dist/catalog/provenance.js.map +1 -0
- package/dist/catalog/runtime.js +2 -0
- package/dist/catalog/runtime.js.map +1 -0
- package/dist/catalog/text.js +140 -0
- package/dist/catalog/text.js.map +1 -0
- package/dist/cli.js +81 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/check.js +155 -0
- package/dist/commands/check.js.map +1 -0
- package/dist/commands/pack.js +182 -0
- package/dist/commands/pack.js.map +1 -0
- package/dist/commands/serve.js +452 -0
- package/dist/commands/serve.js.map +1 -0
- package/dist/config/company-config.js +272 -0
- package/dist/config/company-config.js.map +1 -0
- package/dist/derive/derived-document.js +42 -0
- package/dist/derive/derived-document.js.map +1 -0
- package/dist/engine/qmd-render.js +54 -0
- package/dist/engine/qmd-render.js.map +1 -0
- package/dist/engine/qmd.js +241 -0
- package/dist/engine/qmd.js.map +1 -0
- package/dist/fs/cache-dir.js +140 -0
- package/dist/fs/cache-dir.js.map +1 -0
- package/dist/fs/company-lock.js +162 -0
- package/dist/fs/company-lock.js.map +1 -0
- package/dist/fs/walk.js +168 -0
- package/dist/fs/walk.js.map +1 -0
- package/dist/log.js +65 -0
- package/dist/log.js.map +1 -0
- package/dist/mcp/server.js +30 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/stdio.js +45 -0
- package/dist/mcp/stdio.js.map +1 -0
- package/dist/mcp/tools.js +249 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/report/report.js +43 -0
- package/dist/report/report.js.map +1 -0
- package/dist/search/engine.js +2 -0
- package/dist/search/engine.js.map +1 -0
- package/dist/search/query.js +91 -0
- package/dist/search/query.js.map +1 -0
- package/dist/search/search.js +224 -0
- package/dist/search/search.js.map +1 -0
- package/dist/search/snippet.js +95 -0
- package/dist/search/snippet.js.map +1 -0
- package/dist/serve/poller.js +119 -0
- package/dist/serve/poller.js.map +1 -0
- package/dist/serve/runtime.js +390 -0
- package/dist/serve/runtime.js.map +1 -0
- package/dist/source/git-runner.js +232 -0
- package/dist/source/git-runner.js.map +1 -0
- package/dist/source/git.js +457 -0
- package/dist/source/git.js.map +1 -0
- package/dist/source/local.js +20 -0
- package/dist/source/local.js.map +1 -0
- package/dist/source/source.js +2 -0
- package/dist/source/source.js.map +1 -0
- package/npm-shrinkwrap.json +5482 -0
- package/package.json +80 -0
- package/plugin/claude-code/.claude-plugin/plugin.json +15 -0
- package/plugin/claude-code/.mcp.json +10 -0
- package/plugin/claude-code/skills/okf-catalog/SKILL.md +15 -0
- package/recipes/publish/README.md +32 -0
- package/recipes/publish/checkers.lock +435 -0
- package/recipes/publish/checkers.txt +5 -0
- package/recipes/publish/pack.sh +60 -0
- package/recipes/publish/publish.yml +88 -0
- package/recipes/publish/push.sh +63 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../src/mcp/tools.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAE3B,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAgB,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EACL,mBAAmB,EAEnB,gBAAgB,EAChB,cAAc,EACd,WAAW,EACX,eAAe,EACf,aAAa,EACb,aAAa,EACb,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,GACd,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAU7C,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,IAAI,GAAG,CAAC,IAAY,EAAc,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;AAClG,MAAM,EAAE,GAAG,CAAC,IAAY,EAAE,UAAmC,EAAc,EAAE,CAAC,CAAC;IAC7E,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACjC,iBAAiB,EAAE,UAAU;CAC9B,CAAC,CAAC;AAEH,4CAA4C;AAC5C,MAAM,KAAK,GAAG,CAAC,KAAyB,EAAsB,EAAE,CAC9D,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;AAE9E,iHAAiH;AACjH,MAAM,eAAe,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAE1F,2GAA2G;AAC3G,MAAM,aAAa,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAExF,SAAS,UAAU,CAAC,OAAgB;IAClC,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;SACtC,IAAI,CAAC,UAAU,CAAC;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,KAAK,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;AACzF,CAAC;AAED,SAAS,OAAO,CAAC,UAAsB;IACrC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;IACtC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,IAAI,CACT,iDAAiD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CACxI,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,OAAgB,EAChB,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxD,MAAM,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAClD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC7F,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,OAAgB;IACnC,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACxC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC;QACjD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC;QAC/D,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,QAAQ,CAAC,MAAkB;IAClC,MAAM,IAAI,GACR,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;QACjD,CAAC,CAAC,2DAA2D,MAAM,CAAC,UAAU,GAAG;QACjF,CAAC,CAAC,EAAE,CAAC;IACT,OAAO,GAAG,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;AACvE,CAAC;AAED,MAAM,YAAY,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,GAAG,IAAI,IAAI,aAAa,EAAE,CAAC;AAE1E,oIAAoI;AACpI,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAU,EAAE,CACpD,0EAA0E,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;AAErG;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAiB,EACjB,OAAgB,EAChB,OAAoB,EACpB,KAAiB,EACjB,GAAS;IAET,MAAM,OAAO,GACX,CACE,IAAY,EACZ,EAAyF,EACzF,EAAE,CACJ,KAAK,EAAE,IAAO,EAAuB,EAAE;QACrC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC;QAC3C,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,CAClD,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAC3D,CAAC;YACF,MAAM,IAAI,GAAI,MAAM,CAAC,iBAAsD,EAAE,IAAI,EAAE,MAAM,CAAC;YAC1F,GAAG,EAAE,IAAI,CAAC,WAAW,EAAE;gBACrB,IAAI;gBACJ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBAC3C,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;gBACvC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC;gBACpB,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE;gBACtB,IAAI;gBACJ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBAC3C,KAAK,EAAG,KAAe,CAAC,OAAO;aAChC,CAAC,CAAC;YACH,6GAA6G;YAC7G,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC;YAC9C,IAAI,WAAW,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1E,OAAO,IAAI,CAAC,qEAAqE,CAAC,CAAC;QACrF,CAAC;IACH,CAAC,CAAC;IAEJ,MAAM,CAAC,YAAY,CACjB,QAAQ,EACR;QACE,KAAK,EAAE,6BAA6B;QACpC,WAAW,EAAE,YAAY,CACvB,+QAA+Q,CAChR;QACD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,GAAG,CAAC;iBACR,QAAQ,CAAC,sDAAsD,CAAC;YACnE,IAAI,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,GAAG,CAAC,GAAG,CAAC;iBACR,QAAQ,EAAE;iBACV,QAAQ,CAAC,iDAAiD,CAAC;YAC9D,KAAK,EAAE,CAAC;iBACL,MAAM,EAAE;iBACR,GAAG,CAAC,IAAI,CAAC;iBACT,QAAQ,EAAE;iBACV,QAAQ,CAAC,6CAA6C,CAAC;YAC1D,aAAa,EAAE,CAAC;iBACb,OAAO,EAAE;iBACT,QAAQ,EAAE;iBACV,QAAQ,CAAC,qEAAqE,CAAC;YAClF,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;SACtF,CAAC;QACF,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE;QACnD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,IAAwB,CAAC;QAC7B,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;YACvE,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,OAAO,IAAI,CACT,wBAAwB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,2BAA2B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE,CAC5H,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAyB,CAAC;QAC9B,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;YACrC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/D,OAAO,IAAI,CACT,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,sBAAsB,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CACxG,CAAC;YACJ,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,KAAK,GAAG,SAAS,CAAC;QAC5C,CAAC;QACD,MAAM,GAAG,GAAG,KAAK,EAAE,CAAC;QACpB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAC3B,UAAU,CAAC,OAAO,EAClB,MAAM,EACN;YACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACvC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;YACzC,YAAY,EAAE,IAAI,CAAC,aAAa,IAAI,KAAK;YACzC,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY;SAC1C,EACD,GAAG,CACJ,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAC3C,OAAO,IAAI,CACT,0HAA0H,CAC3H,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;QACtF,OAAO;YACL,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YACjF,SAAS,EAAE,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE;SACxF,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,UAAU,EACV;QACE,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,YAAY,CACvB,yPAAyP,CAC1P;QACD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,IAAI,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,IAAI,CAAC;iBACT,QAAQ,CAAC,0EAA0E,CAAC;YACvF,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,QAAQ,EAAE;iBACV,QAAQ,CAAC,kDAAkD,CAAC;SAChE,CAAC;QACF,YAAY,EAAE,gBAAgB;QAC9B,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,OAAO,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;QACvC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;YACxE,OAAO,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,eAAe,CAC5B,QAAQ,CAAC,IAAI,EACb,QAAQ,CAAC,MAAM,EACf,MAAM,EACN,OAAO,CAAC,YAAY,CACrB,CAAC;YACF,OAAO,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;QACpE,OAAO,IAAI,CACT,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,EAAE,CACtH,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,SAAS,EACT;QACE,KAAK,EAAE,6BAA6B;QACpC,WAAW,EAAE,YAAY,CACvB,yMAAyM,CAC1M;QACD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,GAAG,CAAC,IAAI,CAAC;iBACT,QAAQ,EAAE;iBACV,QAAQ,CAAC,4CAA4C,CAAC;YACzD,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,QAAQ,EAAE;iBACV,QAAQ,CAAC,mDAAmD,CAAC;SACjE,CAAC;QACF,YAAY,EAAE,mBAAmB;QACjC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,OAAO,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC;QACpC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,cAAc,CAC3B,UAAU,CAAC,OAAO,EAClB,MAAM,EACN,IAAI,CAAC,MAAM,IAAI,CAAC,EAChB,OAAO,CAAC,YAAY,CACrB,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,IAAI,CACT,sBAAsB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,sBAAsB,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GACR,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;YACjD,CAAC,CAAC,2DAA2D,MAAM,CAAC,UAAU,GAAG;YACjF,CAAC,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,GAAG,cAAc,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,MAAM,CAAC,MAAM,WAAW,MAAM,CAAC,OAAO,CAAC,MAAM,SAAS,CAAC;QACvI,OAAO,EAAE,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,GAAG,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;IACxE,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,QAAQ,EACR;QACE,KAAK,EAAE,2BAA2B;QAClC,WAAW,EAAE,YAAY,CACvB,0LAA0L,CAC3L;QACD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC,CAAC,CACH,CAAC;AACJ,CAAC","sourcesContent":["import type { McpServer } from \"@modelcontextprotocol/server\";\nimport { z } from \"zod/v4\";\nimport type { ReservedFile } from \"../bundle/model.js\";\nimport { byCodeUnit } from \"../bundle/paths.js\";\nimport { type Catalog, listTypes } from \"../catalog/model.js\";\nimport { nearestPaths } from \"../catalog/nearest.js\";\nimport {\n CatalogOutputSchema,\n type PageOutput,\n PageOutputSchema,\n projectCatalog,\n projectPage,\n projectReserved,\n projectSearch,\n projectStatus,\n refusingText,\n SearchOutputSchema,\n StatusOutputSchema,\n statusSummary,\n} from \"../catalog/outputs.js\";\nimport type { Generation, Runtime, ToolOptions } from \"../catalog/runtime.js\";\nimport { DATA_SENTENCE, safe } from \"../catalog/text.js\";\nimport type { Log } from \"../log.js\";\nimport type { Engine } from \"../search/engine.js\";\nimport { search } from \"../search/search.js\";\n\ntype ToolResult = {\n content: Array<{ type: \"text\"; text: string }>;\n structuredContent?: Record<string, unknown>;\n isError?: boolean;\n /** Extra fields for the `tool.call` log record, stripped before the result leaves. */\n logFields?: Record<string, number>;\n};\n\nconst FOLDER_LIST_CAP = 50;\n\nconst fail = (text: string): ToolResult => ({ content: [{ type: \"text\", text }], isError: true });\nconst ok = (text: string, structured: Record<string, unknown>): ToolResult => ({\n content: [{ type: \"text\", text }],\n structuredContent: structured,\n});\n\n/** A blank optional string means absent. */\nconst blank = (value: string | undefined): string | undefined =>\n value === undefined || value.trim().length === 0 ? undefined : value.trim();\n\n/** A folder as the search filter normalises it: no leading or trailing slashes; the empty string is the root. */\nconst normaliseFolder = (value: string): string => value.trim().replace(/^\\/+|\\/+$/g, \"\");\n\n/** A page path as the model may write it: one leading `/` or `./` stripped (OKF bundle-absolute links). */\nconst normalisePath = (value: string): string => value.trim().replace(/^(\\.\\/|\\/)/, \"\");\n\nfunction folderList(catalog: Catalog): string {\n const names = [...catalog.folders.keys()]\n .sort(byCodeUnit)\n .map((f) => (f === \"\" ? \"(root)\" : safe(f)));\n const shown = names.slice(0, FOLDER_LIST_CAP).join(\", \");\n return names.length > FOLDER_LIST_CAP ? `${shown} … (${names.length} folders)` : shown;\n}\n\nfunction refused(generation: Generation): ToolResult | undefined {\n const fatal = generation.report.fatal;\n if (fatal === undefined) return undefined;\n return fail(\n `the bundle was refused and nothing is served: ${safe(fatal.rule)}${fatal.path ? ` (${safe(fatal.path)})` : \"\"}: ${safe(fatal.detail)}`,\n );\n}\n\nfunction reservedAt(\n catalog: Catalog,\n path: string,\n): { file: ReservedFile; source: \"file\" | \"generated\" } | undefined {\n const slash = path.lastIndexOf(\"/\");\n const folder = slash === -1 ? \"\" : path.slice(0, slash);\n const name = slash === -1 ? path : path.slice(slash + 1);\n const entry = catalog.folders.get(folder);\n if (entry === undefined) return undefined;\n if (name === \"index.md\" && entry.index !== undefined)\n return { file: entry.index, source: entry.indexSource };\n if (name === \"log.md\" && entry.log !== undefined) return { file: entry.log, source: \"file\" };\n return undefined;\n}\n\nfunction servedPaths(catalog: Catalog): string[] {\n const paths = [...catalog.pages.keys()];\n for (const [folder, entry] of catalog.folders) {\n const prefix = folder === \"\" ? \"\" : `${folder}/`;\n if (entry.index !== undefined) paths.push(`${prefix}index.md`);\n if (entry.log !== undefined) paths.push(`${prefix}log.md`);\n }\n return paths;\n}\n\nfunction pageText(output: PageOutput): string {\n const tail =\n output.truncated && output.nextOffset !== undefined\n ? `\\n[truncated at the result budget; continue with offset ${output.nextOffset}]`\n : \"\";\n return `${output.citation}\\n${output.notice}\\n${output.body}${tail}`;\n}\n\nconst describeType = (text: string): string => `${text} ${DATA_SENTENCE}`;\n\n/** The one sentence every tool answers with while the server cannot serve: the fix, never a path the model has no business with. */\nconst refusingSentence = (refusing: string): string =>\n `the server is refusing every request until its configuration is fixed: ${refusingText(refusing)}`;\n\n/**\n * The four tools. Every handler runs under a lease on the current generation, answers in both channels, and\n * turns anything it cannot repair into a fixed sentence; the detail goes to the log, never to the model.\n */\nexport function registerTools(\n server: McpServer,\n runtime: Runtime,\n options: ToolOptions,\n clock: () => Date,\n log?: Log,\n): void {\n const guarded =\n <A>(\n tool: string,\n fn: (args: A, generation: Generation, engine: Engine) => Promise<ToolResult> | ToolResult,\n ) =>\n async (args: A): Promise<ToolResult> => {\n const started = performance.now();\n const refusing = runtime.status().refusing;\n if (refusing !== undefined) return fail(refusingSentence(refusing));\n try {\n const { logFields, ...result } = await runtime.lease<ToolResult>(\n async (generation, engine) => fn(args, generation, engine),\n );\n const hits = (result.structuredContent as { hits?: unknown[] } | undefined)?.hits?.length;\n log?.info(\"tool.call\", {\n tool,\n ms: Math.round(performance.now() - started),\n ...(hits === undefined ? {} : { hits }),\n ...(logFields ?? {}),\n ...(result.isError === true ? { error: \"answered-with-error\" } : {}),\n });\n return result;\n } catch (error) {\n log?.error(\"tool.call\", {\n tool,\n ms: Math.round(performance.now() - started),\n error: (error as Error).message,\n });\n // The call that started the first load is the one that sees it fail: answer with the fix, as later calls do.\n const refusingNow = runtime.status().refusing;\n if (refusingNow !== undefined) return fail(refusingSentence(refusingNow));\n return fail(\"the server hit a defect answering this call; its log has the detail\");\n }\n };\n\n server.registerTool(\n \"search\",\n {\n title: \"Search the knowledge bundle\",\n description: describeType(\n \"Finds pages by keywords. Write one concept per word; common words are dropped, and when no page holds every word the match is relaxed and the result says so. Each hit carries its path, type, status, trust tier, recheck date, source count, resource and a quoted snippet.\",\n ),\n inputSchema: z.object({\n question: z\n .string()\n .min(1)\n .max(200)\n .describe(\"Keywords, or a short question; one concept per word.\"),\n type: z\n .string()\n .max(200)\n .optional()\n .describe(\"Only pages of this type (case does not matter).\"),\n topic: z\n .string()\n .max(1024)\n .optional()\n .describe(\"Only pages under this folder of the bundle.\"),\n include_stale: z\n .boolean()\n .optional()\n .describe(\"Include pages past their recheck date; they are flagged as overdue.\"),\n limit: z.number().int().min(1).max(25).optional().describe(\"How many hits, 1 to 25.\"),\n }),\n outputSchema: SearchOutputSchema,\n annotations: { readOnlyHint: true },\n },\n guarded(\"search\", async (args, generation, engine) => {\n const stop = refused(generation);\n if (stop !== undefined) return stop;\n const wantedType = blank(args.type);\n let type: string | undefined;\n if (wantedType !== undefined) {\n const types = listTypes(generation.catalog);\n type = types.find((t) => t.toLowerCase() === wantedType.toLowerCase());\n if (type === undefined) {\n return fail(\n `no page has the type ${JSON.stringify(safe(wantedType))}; the types in use are: ${types.map(safe).join(\", \") || \"(none)\"}`,\n );\n }\n }\n const wantedTopic = blank(args.topic);\n let topic: string | undefined;\n if (wantedTopic !== undefined) {\n topic = normaliseFolder(wantedTopic);\n if (topic.length > 0 && !generation.catalog.folders.has(topic)) {\n return fail(\n `there is no folder ${JSON.stringify(safe(topic))}; the folders are: ${folderList(generation.catalog)}`,\n );\n }\n if (topic.length === 0) topic = undefined;\n }\n const now = clock();\n const response = await search(\n generation.catalog,\n engine,\n {\n question: args.question,\n ...(type === undefined ? {} : { type }),\n ...(topic === undefined ? {} : { topic }),\n includeStale: args.include_stale ?? false,\n limit: args.limit ?? options.limitDefault,\n },\n now,\n );\n if (response.reason === \"no-content-terms\") {\n return fail(\n \"every word of the question is a common word the index ignores; ask with keywords, the distinctive words a page would use\",\n );\n }\n const output = projectSearch(response, generation.catalog, now, { dev: options.dev });\n return {\n ...ok([output.summary, ...output.hits.map((h) => h.citation)].join(\"\\n\"), output),\n logFields: { engineQueries: response.engineQueries, rowsFetched: response.rowsFetched },\n };\n }),\n );\n\n server.registerTool(\n \"get_page\",\n {\n title: \"Read a page\",\n description: describeType(\n \"Returns one page whole, with its provenance header first: path, type, status, trust tier, verifier, recheck date and deprecation. Reserved files (index.md, log.md) are served too. A long page is cut at the result budget and says where to continue.\",\n ),\n inputSchema: z.object({\n path: z\n .string()\n .min(1)\n .max(1024)\n .describe(\"The page's path in the bundle, as a search result or a catalog lists it.\"),\n offset: z\n .number()\n .int()\n .min(0)\n .optional()\n .describe(\"Continue a long page from this character offset.\"),\n }),\n outputSchema: PageOutputSchema,\n annotations: { readOnlyHint: true },\n },\n guarded(\"get_page\", (args, generation) => {\n const stop = refused(generation);\n if (stop !== undefined) return stop;\n const path = normalisePath(args.path);\n const offset = args.offset ?? 0;\n const page = generation.catalog.pages.get(path);\n if (page !== undefined) {\n const output = projectPage(page, clock(), offset, options.resultBudget);\n return ok(pageText(output), output);\n }\n const reserved = reservedAt(generation.catalog, path);\n if (reserved !== undefined) {\n const output = projectReserved(\n reserved.file,\n reserved.source,\n offset,\n options.resultBudget,\n );\n return ok(pageText(output), output);\n }\n const nearest = nearestPaths(servedPaths(generation.catalog), path);\n return fail(\n `no page at ${JSON.stringify(safe(path))}; the nearest served paths are: ${nearest.map(safe).join(\", \") || \"(none)\"}`,\n );\n }),\n );\n\n server.registerTool(\n \"catalog\",\n {\n title: \"List a folder of the bundle\",\n description: describeType(\n \"Returns a folder's index: its pages with their titles and descriptions, and the folder's index text as the company wrote it or as the server generated it. Start here, at the root, to see what exists.\",\n ),\n inputSchema: z.object({\n folder: z\n .string()\n .max(1024)\n .optional()\n .describe(\"A folder of the bundle; omit for the root.\"),\n offset: z\n .number()\n .int()\n .min(0)\n .optional()\n .describe(\"Continue a long index from this character offset.\"),\n }),\n outputSchema: CatalogOutputSchema,\n annotations: { readOnlyHint: true },\n },\n guarded(\"catalog\", (args, generation) => {\n const stop = refused(generation);\n if (stop !== undefined) return stop;\n const folder = normaliseFolder(args.folder ?? \"\");\n const output = projectCatalog(\n generation.catalog,\n folder,\n args.offset ?? 0,\n options.resultBudget,\n );\n if (output === undefined) {\n return fail(\n `there is no folder ${JSON.stringify(safe(folder))}; the folders are: ${folderList(generation.catalog)}`,\n );\n }\n const tail =\n output.truncated && output.nextOffset !== undefined\n ? `\\n[truncated at the result budget; continue with offset ${output.nextOffset}]`\n : \"\";\n const head = `catalog of ${folder === \"\" ? \"the bundle root\" : safe(folder)} (${output.source} index, ${output.entries.length} pages)`;\n return ok(`${head}\\n${output.notice}\\n${output.text}${tail}`, output);\n }),\n );\n\n server.registerTool(\n \"status\",\n {\n title: \"Report the bundle's state\",\n description: describeType(\n \"Reports what was loaded: counts of pages admitted, refused and degraded, the lists the report carries, the integrity mode, the engine's counts and the lock. Nothing in it is page text.\",\n ),\n inputSchema: z.object({}),\n outputSchema: StatusOutputSchema,\n annotations: { readOnlyHint: true },\n },\n guarded(\"status\", (_args, generation) => {\n const output = projectStatus(generation, runtime.status(), options, clock());\n return ok(statusSummary(output), output);\n }),\n );\n}\n"]}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { escapeControls } from "../catalog/text.js";
|
|
2
|
+
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
3
|
+
/** The report as text for `status` and `check`: the fatal refusal first when there is one, then counts, then detail. */
|
|
4
|
+
export function renderReport(report) {
|
|
5
|
+
const lines = [];
|
|
6
|
+
if (report.fatal !== undefined) {
|
|
7
|
+
lines.push(`FATAL ${report.fatal.rule}${report.fatal.path ? ` (${report.fatal.path})` : ""}: ${report.fatal.detail}`);
|
|
8
|
+
}
|
|
9
|
+
lines.push(`${plural(report.admitted, "page")} admitted; ${report.excludedByStatus} excluded by status; ${plural(report.attachments, "attachment")}; ${report.hidden} hidden; ${report.refusals.length} refused; ${report.degradations.length} degraded${report.commit ? `; commit ${report.commit.slice(0, 12)}` : ""}`);
|
|
10
|
+
if (report.refusals.length > 0) {
|
|
11
|
+
lines.push("Refused:");
|
|
12
|
+
for (const r of report.refusals)
|
|
13
|
+
lines.push(` ${r.path}: ${r.rule} (${r.detail})`);
|
|
14
|
+
}
|
|
15
|
+
if (report.integrity === "skipped") {
|
|
16
|
+
lines.push("Integrity was not checked: the manifest was not required for this load.");
|
|
17
|
+
}
|
|
18
|
+
if (report.degradations.length > 0) {
|
|
19
|
+
lines.push("Degraded:");
|
|
20
|
+
for (const d of report.degradations)
|
|
21
|
+
lines.push(` ${d.path}: ${d.code} (${d.detail})`);
|
|
22
|
+
}
|
|
23
|
+
if (report.unknownTypes.length > 0)
|
|
24
|
+
lines.push(`Undeclared types: ${report.unknownTypes.join(", ")}`);
|
|
25
|
+
if (report.unknownStatuses.length > 0) {
|
|
26
|
+
lines.push(`Unknown statuses: ${report.unknownStatuses.map((s) => `${s.path} (${s.value})`).join(", ")}`);
|
|
27
|
+
}
|
|
28
|
+
if (report.brokenLinks.length > 0)
|
|
29
|
+
lines.push(`Broken links: ${report.brokenLinks.map((l) => `${l.from} → ${l.raw}`).join(", ")}`);
|
|
30
|
+
if (report.linksToUnserved.length > 0) {
|
|
31
|
+
lines.push(`Links to pages not served: ${report.linksToUnserved.map((l) => `${l.from} → ${l.target}`).join(", ")}`);
|
|
32
|
+
}
|
|
33
|
+
if (report.missingOnDisk.length > 0)
|
|
34
|
+
lines.push(`Listed in the manifest but missing: ${report.missingOnDisk.join(", ")}`);
|
|
35
|
+
if (report.encodedFolders.length > 0) {
|
|
36
|
+
lines.push(`Folders renamed for the search engine: ${report.encodedFolders.join(", ")}`);
|
|
37
|
+
}
|
|
38
|
+
if (report.foldersWithoutIndex.length > 0) {
|
|
39
|
+
lines.push(`Folders without an index (generated): ${report.foldersWithoutIndex.map((f) => f || "(root)").join(", ")}`);
|
|
40
|
+
}
|
|
41
|
+
return `${lines.map(escapeControls).join("\n")}\n`;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/report/report.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,IAAY,EAAU,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AAExF,wHAAwH;AACxH,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CACR,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAC1G,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CACR,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,cAAc,MAAM,CAAC,gBAAgB,wBAAwB,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,KAAK,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,MAAM,CAAC,YAAY,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9S,CAAC;IACF,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpE,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CACR,qBAAqB,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC9F,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClG,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CACR,8BAA8B,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxG,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,uCAAuC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvF,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,MAAM,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,KAAK,CAAC,IAAI,CACR,yCAAyC,MAAM,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC3G,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrD,CAAC","sourcesContent":["import type { Report } from \"../bundle/model.js\";\nimport { escapeControls } from \"../catalog/text.js\";\n\nconst plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? \"\" : \"s\"}`;\n\n/** The report as text for `status` and `check`: the fatal refusal first when there is one, then counts, then detail. */\nexport function renderReport(report: Report): string {\n const lines: string[] = [];\n if (report.fatal !== undefined) {\n lines.push(\n `FATAL ${report.fatal.rule}${report.fatal.path ? ` (${report.fatal.path})` : \"\"}: ${report.fatal.detail}`,\n );\n }\n lines.push(\n `${plural(report.admitted, \"page\")} admitted; ${report.excludedByStatus} excluded by status; ${plural(report.attachments, \"attachment\")}; ${report.hidden} hidden; ${report.refusals.length} refused; ${report.degradations.length} degraded${report.commit ? `; commit ${report.commit.slice(0, 12)}` : \"\"}`,\n );\n if (report.refusals.length > 0) {\n lines.push(\"Refused:\");\n for (const r of report.refusals) lines.push(` ${r.path}: ${r.rule} (${r.detail})`);\n }\n if (report.integrity === \"skipped\") {\n lines.push(\"Integrity was not checked: the manifest was not required for this load.\");\n }\n if (report.degradations.length > 0) {\n lines.push(\"Degraded:\");\n for (const d of report.degradations) lines.push(` ${d.path}: ${d.code} (${d.detail})`);\n }\n if (report.unknownTypes.length > 0)\n lines.push(`Undeclared types: ${report.unknownTypes.join(\", \")}`);\n if (report.unknownStatuses.length > 0) {\n lines.push(\n `Unknown statuses: ${report.unknownStatuses.map((s) => `${s.path} (${s.value})`).join(\", \")}`,\n );\n }\n if (report.brokenLinks.length > 0)\n lines.push(`Broken links: ${report.brokenLinks.map((l) => `${l.from} → ${l.raw}`).join(\", \")}`);\n if (report.linksToUnserved.length > 0) {\n lines.push(\n `Links to pages not served: ${report.linksToUnserved.map((l) => `${l.from} → ${l.target}`).join(\", \")}`,\n );\n }\n if (report.missingOnDisk.length > 0)\n lines.push(`Listed in the manifest but missing: ${report.missingOnDisk.join(\", \")}`);\n if (report.encodedFolders.length > 0) {\n lines.push(`Folders renamed for the search engine: ${report.encodedFolders.join(\", \")}`);\n }\n if (report.foldersWithoutIndex.length > 0) {\n lines.push(\n `Folders without an index (generated): ${report.foldersWithoutIndex.map((f) => f || \"(root)\").join(\", \")}`,\n );\n }\n return `${lines.map(escapeControls).join(\"\\n\")}\\n`;\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../src/search/engine.ts"],"names":[],"mappings":"","sourcesContent":["import type { PagePath } from \"../bundle/model.js\";\nimport type { DerivedDocument } from \"../derive/derived-document.js\";\n\nexport interface EngineHit {\n path: PagePath;\n /** qmd's score in [0, 1). */\n score: number;\n /** The raw BM25 value recovered from the score: on one scale for every query, and additive across terms. */\n bm25: number;\n}\n\nexport interface IndexResult {\n documents: number;\n indexed: number;\n updated: number;\n unchanged: number;\n removed: number;\n skipped: number;\n /** Rendered documents the engine did not index, by bundle path: a path that collided, could not be written, or was skipped by the engine. */\n notIndexed: PagePath[];\n /** Paths that name the same file once case and Unicode form are ignored; the first by path order is kept. */\n collisions: Array<{ kept: PagePath; dropped: PagePath }>;\n /** Bundle folders whose names had to be encoded for the engine. */\n encodedFolders: string[];\n}\n\n/** The search engine as the OKF layer sees it. The adapter owns files, paths and the engine's conventions. */\nexport interface Engine {\n index(docs: readonly DerivedDocument[]): Promise<IndexResult>;\n /** Every term must match as a prefix; `limit` is exact; hits come best first. */\n lex(terms: readonly string[], limit: number): Promise<EngineHit[]>;\n status(): Promise<{ documents: number }>;\n close(): Promise<void>;\n}\n"]}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* From a question to the terms the engine receives. qmd ANDs every term as a prefix, treats a leading hyphen as
|
|
3
|
+
* a negation, and turns a run of Chinese, Japanese or Korean characters into one exact phrase, so the question
|
|
4
|
+
* is cut into safe tokens here: letters and digits with internal hyphens, two or more characters, stopwords and
|
|
5
|
+
* question words removed, CJK runs split into overlapping pairs so the relaxed rung has something to relax.
|
|
6
|
+
*/
|
|
7
|
+
export const STOPWORDS = new Set(("a an the and or but nor so yet if then else than as of at by for from in into on onto to with without within " +
|
|
8
|
+
"about above across after against along among around before behind below beneath beside between beyond during " +
|
|
9
|
+
"except inside near off out outside over past since through throughout toward towards under until up upon via " +
|
|
10
|
+
"i me my mine we us our ours you your yours he him his she her hers it its they them their theirs this that these " +
|
|
11
|
+
"those who whom whose which what when where why how whether while each every all any both few more most other " +
|
|
12
|
+
"some such no not only own same too very just also ever never here there now again further once " +
|
|
13
|
+
"am is are was were be been being have has had having do does did doing done will would shall should can could " +
|
|
14
|
+
"may might must ought get got gets getting give gives tell tells show shows let lets please thanks " +
|
|
15
|
+
"something anything nothing everything someone anyone " +
|
|
16
|
+
// Residues of English contractions, in case one arrives on its own (`re: budget`).
|
|
17
|
+
"ve re ll don doesn didn isn aren wasn weren hasn haven hadn wouldn shouldn couldn mustn").split(/\s+/));
|
|
18
|
+
const MAX_TERMS = 12;
|
|
19
|
+
const TOKEN = /^[\p{L}\p{N}\p{M}]+(?:-[\p{L}\p{N}\p{M}]+)*$/u;
|
|
20
|
+
const IRREGULAR_CONTRACTIONS = {
|
|
21
|
+
"can't": "can",
|
|
22
|
+
"won't": "will",
|
|
23
|
+
"shan't": "shall",
|
|
24
|
+
};
|
|
25
|
+
/** `don't` becomes `do`, `we've` becomes `we`: the residue of a contraction must never become a content term. */
|
|
26
|
+
function uncontract(word) {
|
|
27
|
+
const plain = word.replace(/’/g, "'");
|
|
28
|
+
const irregular = IRREGULAR_CONTRACTIONS[plain];
|
|
29
|
+
if (irregular !== undefined)
|
|
30
|
+
return irregular;
|
|
31
|
+
return plain.replace(/n't$/, "").replace(/'(s|re|ve|ll|d|m)$/, "");
|
|
32
|
+
}
|
|
33
|
+
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
|
34
|
+
const CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+|[^\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
|
|
35
|
+
function pairs(run) {
|
|
36
|
+
const chars = [...run];
|
|
37
|
+
if (chars.length < 2)
|
|
38
|
+
return chars;
|
|
39
|
+
const out = [];
|
|
40
|
+
for (let i = 0; i + 1 < chars.length; i++)
|
|
41
|
+
out.push(`${chars[i] ?? ""}${chars[i + 1] ?? ""}`);
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Lower-cased content tokens in question order; CJK runs become overlapping pairs; nothing shorter than two code
|
|
46
|
+
* points. A run of two or more hyphens separates tokens like any other punctuation. Digits are split exactly as
|
|
47
|
+
* the engine's tokenizer splits them, so `1,000` becomes `1` (dropped) and `000`, which is what the index holds.
|
|
48
|
+
* Combining marks stay with their letters, as the engine keeps them. English contractions lose their suffix.
|
|
49
|
+
*/
|
|
50
|
+
export function tokenize(question) {
|
|
51
|
+
const tokens = [];
|
|
52
|
+
const words = question
|
|
53
|
+
.toLowerCase()
|
|
54
|
+
.split(/[^\p{L}\p{N}\p{M}'’-]+|-{2,}/u)
|
|
55
|
+
.flatMap((raw) => uncontract(raw.replace(/^['’]+|['’]+$/g, "")).split(/['’]+/));
|
|
56
|
+
for (const raw of words) {
|
|
57
|
+
const trimmed = raw.replace(/^-+|-+$/g, "");
|
|
58
|
+
if (trimmed.length === 0)
|
|
59
|
+
continue;
|
|
60
|
+
if (CJK.test(trimmed)) {
|
|
61
|
+
for (const run of trimmed.match(CJK_RUN) ?? []) {
|
|
62
|
+
if (CJK.test(run))
|
|
63
|
+
tokens.push(...pairs(run));
|
|
64
|
+
else if (TOKEN.test(run) && [...run].length >= 2)
|
|
65
|
+
tokens.push(run);
|
|
66
|
+
}
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (!TOKEN.test(trimmed) || [...trimmed].length < 2)
|
|
70
|
+
continue;
|
|
71
|
+
tokens.push(trimmed);
|
|
72
|
+
}
|
|
73
|
+
return tokens;
|
|
74
|
+
}
|
|
75
|
+
export function normaliseQuestion(question) {
|
|
76
|
+
const terms = [];
|
|
77
|
+
const dropped = [];
|
|
78
|
+
for (const token of tokenize(question)) {
|
|
79
|
+
if (STOPWORDS.has(token)) {
|
|
80
|
+
dropped.push(token);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (terms.includes(token))
|
|
84
|
+
continue;
|
|
85
|
+
terms.push(token);
|
|
86
|
+
}
|
|
87
|
+
// Terms past the twelfth are not sent; they are reported as dropped so a cut question never looks complete.
|
|
88
|
+
dropped.push(...terms.slice(MAX_TERMS));
|
|
89
|
+
return { terms: terms.slice(0, MAX_TERMS), dropped };
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/search/query.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,SAAS,GAAwB,IAAI,GAAG,CACnD,CACE,+GAA+G;IAC/G,+GAA+G;IAC/G,+GAA+G;IAC/G,mHAAmH;IACnH,+GAA+G;IAC/G,iGAAiG;IACjG,gHAAgH;IAChH,oGAAoG;IACpG,uDAAuD;IACvD,mFAAmF;IACnF,yFAAyF,CAC1F,CAAC,KAAK,CAAC,KAAK,CAAC,CACf,CAAC;AAEF,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,KAAK,GAAG,+CAA+C,CAAC;AAC9D,MAAM,sBAAsB,GAA2B;IACrD,OAAO,EAAE,KAAK;IACd,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,OAAO;CAClB,CAAC;AAEF,iHAAiH;AACjH,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AACD,MAAM,GAAG,GAAG,0EAA0E,CAAC;AACvF,MAAM,OAAO,GACX,sJAAsJ,CAAC;AAEzJ,SAAS,KAAK,CAAC,GAAW;IACxB,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,QAAgB;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,QAAQ;SACnB,WAAW,EAAE;SACb,KAAK,CAAC,+BAA+B,CAAC;SACtC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACnC,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC/C,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;qBACzC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC9D,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IACD,4GAA4G;IAC5G,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IACxC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC;AACvD,CAAC","sourcesContent":["/**\n * From a question to the terms the engine receives. qmd ANDs every term as a prefix, treats a leading hyphen as\n * a negation, and turns a run of Chinese, Japanese or Korean characters into one exact phrase, so the question\n * is cut into safe tokens here: letters and digits with internal hyphens, two or more characters, stopwords and\n * question words removed, CJK runs split into overlapping pairs so the relaxed rung has something to relax.\n */\nexport const STOPWORDS: ReadonlySet<string> = new Set(\n (\n \"a an the and or but nor so yet if then else than as of at by for from in into on onto to with without within \" +\n \"about above across after against along among around before behind below beneath beside between beyond during \" +\n \"except inside near off out outside over past since through throughout toward towards under until up upon via \" +\n \"i me my mine we us our ours you your yours he him his she her hers it its they them their theirs this that these \" +\n \"those who whom whose which what when where why how whether while each every all any both few more most other \" +\n \"some such no not only own same too very just also ever never here there now again further once \" +\n \"am is are was were be been being have has had having do does did doing done will would shall should can could \" +\n \"may might must ought get got gets getting give gives tell tells show shows let lets please thanks \" +\n \"something anything nothing everything someone anyone \" +\n // Residues of English contractions, in case one arrives on its own (`re: budget`).\n \"ve re ll don doesn didn isn aren wasn weren hasn haven hadn wouldn shouldn couldn mustn\"\n ).split(/\\s+/),\n);\n\nconst MAX_TERMS = 12;\nconst TOKEN = /^[\\p{L}\\p{N}\\p{M}]+(?:-[\\p{L}\\p{N}\\p{M}]+)*$/u;\nconst IRREGULAR_CONTRACTIONS: Record<string, string> = {\n \"can't\": \"can\",\n \"won't\": \"will\",\n \"shan't\": \"shall\",\n};\n\n/** `don't` becomes `do`, `we've` becomes `we`: the residue of a contraction must never become a content term. */\nfunction uncontract(word: string): string {\n const plain = word.replace(/’/g, \"'\");\n const irregular = IRREGULAR_CONTRACTIONS[plain];\n if (irregular !== undefined) return irregular;\n return plain.replace(/n't$/, \"\").replace(/'(s|re|ve|ll|d|m)$/, \"\");\n}\nconst CJK = /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]/u;\nconst CJK_RUN =\n /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]+|[^\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]+/gu;\n\nfunction pairs(run: string): string[] {\n const chars = [...run];\n if (chars.length < 2) return chars;\n const out: string[] = [];\n for (let i = 0; i + 1 < chars.length; i++) out.push(`${chars[i] ?? \"\"}${chars[i + 1] ?? \"\"}`);\n return out;\n}\n\n/**\n * Lower-cased content tokens in question order; CJK runs become overlapping pairs; nothing shorter than two code\n * points. A run of two or more hyphens separates tokens like any other punctuation. Digits are split exactly as\n * the engine's tokenizer splits them, so `1,000` becomes `1` (dropped) and `000`, which is what the index holds.\n * Combining marks stay with their letters, as the engine keeps them. English contractions lose their suffix.\n */\nexport function tokenize(question: string): string[] {\n const tokens: string[] = [];\n const words = question\n .toLowerCase()\n .split(/[^\\p{L}\\p{N}\\p{M}'’-]+|-{2,}/u)\n .flatMap((raw) => uncontract(raw.replace(/^['’]+|['’]+$/g, \"\")).split(/['’]+/));\n for (const raw of words) {\n const trimmed = raw.replace(/^-+|-+$/g, \"\");\n if (trimmed.length === 0) continue;\n if (CJK.test(trimmed)) {\n for (const run of trimmed.match(CJK_RUN) ?? []) {\n if (CJK.test(run)) tokens.push(...pairs(run));\n else if (TOKEN.test(run) && [...run].length >= 2) tokens.push(run);\n }\n continue;\n }\n if (!TOKEN.test(trimmed) || [...trimmed].length < 2) continue;\n tokens.push(trimmed);\n }\n return tokens;\n}\n\nexport function normaliseQuestion(question: string): { terms: string[]; dropped: string[] } {\n const terms: string[] = [];\n const dropped: string[] = [];\n for (const token of tokenize(question)) {\n if (STOPWORDS.has(token)) {\n dropped.push(token);\n continue;\n }\n if (terms.includes(token)) continue;\n terms.push(token);\n }\n // Terms past the twelfth are not sent; they are reported as dropped so a cut question never looks complete.\n dropped.push(...terms.slice(MAX_TERMS));\n return { terms: terms.slice(0, MAX_TERMS), dropped };\n}\n"]}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import { byCodeUnit } from "../bundle/paths.js";
|
|
2
|
+
import { isOverdue } from "../catalog/provenance.js";
|
|
3
|
+
import { normaliseQuestion, tokenize } from "./query.js";
|
|
4
|
+
const POOL_FACTOR = 4;
|
|
5
|
+
const POOL_CAP = 500;
|
|
6
|
+
const LIMIT_CAP = 25;
|
|
7
|
+
const RELAXED_FLOOR = 0.01;
|
|
8
|
+
/** Below this, a term's best BM25 is SQLite's floored inverse document frequency: the term is in at least half the pages. */
|
|
9
|
+
const FREQUENCY_FLOOR = 1e-3;
|
|
10
|
+
const TIE = 1e-9;
|
|
11
|
+
const TRUST_RANK = {
|
|
12
|
+
"human-reviewed": 0,
|
|
13
|
+
"machine-confirmed": 1,
|
|
14
|
+
unverified: 2,
|
|
15
|
+
};
|
|
16
|
+
function topicPrefix(topic) {
|
|
17
|
+
if (topic === undefined)
|
|
18
|
+
return undefined;
|
|
19
|
+
const trimmed = topic.replace(/^\/+|\/+$/g, "");
|
|
20
|
+
return trimmed.length === 0 ? undefined : `${trimmed}/`;
|
|
21
|
+
}
|
|
22
|
+
function order(a, b) {
|
|
23
|
+
if (Math.abs(b.score - a.score) > TIE)
|
|
24
|
+
return b.score - a.score;
|
|
25
|
+
const trust = TRUST_RANK[a.page.trust] - TRUST_RANK[b.page.trust];
|
|
26
|
+
return trust !== 0 ? trust : byCodeUnit(a.page.path, b.page.path);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Asks the engine for `pool` rows and completes the tie group at the cut. qmd orders equal scores by insertion
|
|
30
|
+
* order, which differs from one index build to the next, so a pool that cuts inside a group of equal scores
|
|
31
|
+
* would make the answer depend on the build. One extra row shows whether the cut fell inside a group; when it
|
|
32
|
+
* did, the request widens until a row below the cut's score is seen, the engine runs out, or the cap is
|
|
33
|
+
* reached. Rows below the cut's score are dropped. At the cap the group is cut as the engine cut it, which is
|
|
34
|
+
* the one residual the engine's order can still reach.
|
|
35
|
+
*/
|
|
36
|
+
async function lexComplete(engine, terms, pool, cost) {
|
|
37
|
+
let ask = pool + 1;
|
|
38
|
+
for (;;) {
|
|
39
|
+
const rows = await engine.lex(terms, ask);
|
|
40
|
+
cost.queries += 1;
|
|
41
|
+
cost.rows += rows.length;
|
|
42
|
+
if (rows.length <= pool)
|
|
43
|
+
return { hits: rows, exhausted: true };
|
|
44
|
+
const cut = rows[pool - 1].bm25;
|
|
45
|
+
let end = pool;
|
|
46
|
+
while (end < rows.length && Math.abs(rows[end].bm25 - cut) <= TIE)
|
|
47
|
+
end += 1;
|
|
48
|
+
if (end < rows.length)
|
|
49
|
+
return { hits: rows.slice(0, end), exhausted: false };
|
|
50
|
+
if (rows.length < ask)
|
|
51
|
+
return { hits: rows, exhausted: true };
|
|
52
|
+
if (ask > POOL_CAP)
|
|
53
|
+
return { hits: rows, exhausted: false };
|
|
54
|
+
ask = Math.min(ask * POOL_FACTOR, POOL_CAP + 1);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Searches the catalog through the engine. The first rung sends every content term, plus the topic's path
|
|
59
|
+
* segments and the type value, as one query, widening the pool while the filters leave it short and the engine
|
|
60
|
+
* returned a full pool. When still short, the relaxed rung sends one query per content term and fuses by
|
|
61
|
+
* summed BM25, ranked by terms matched. Every engine query completes the tie group at its cut, so the answer
|
|
62
|
+
* does not depend on the engine's order among equal scores. Hits carry provenance and their rung; the response
|
|
63
|
+
* says what was filtered and why.
|
|
64
|
+
*/
|
|
65
|
+
export async function search(catalog, engine, request, now) {
|
|
66
|
+
const { terms, dropped } = normaliseQuestion(request.question);
|
|
67
|
+
const limit = Number.isFinite(request.limit)
|
|
68
|
+
? Math.min(LIMIT_CAP, Math.max(1, Math.floor(request.limit)))
|
|
69
|
+
: 1;
|
|
70
|
+
const removed = {
|
|
71
|
+
type: new Set(),
|
|
72
|
+
topic: new Set(),
|
|
73
|
+
stale: new Set(),
|
|
74
|
+
unknown: new Set(),
|
|
75
|
+
};
|
|
76
|
+
const filteredOut = () => ({
|
|
77
|
+
type: removed.type.size,
|
|
78
|
+
topic: removed.topic.size,
|
|
79
|
+
stale: removed.stale.size,
|
|
80
|
+
unknown: removed.unknown.size,
|
|
81
|
+
});
|
|
82
|
+
if (terms.length === 0) {
|
|
83
|
+
return {
|
|
84
|
+
hits: [],
|
|
85
|
+
strategy: "none",
|
|
86
|
+
reason: "no-content-terms",
|
|
87
|
+
terms,
|
|
88
|
+
dropped,
|
|
89
|
+
floored: [],
|
|
90
|
+
considered: 0,
|
|
91
|
+
filteredOut: filteredOut(),
|
|
92
|
+
pool: 0,
|
|
93
|
+
engineQueries: 0,
|
|
94
|
+
rowsFetched: 0,
|
|
95
|
+
topicExhausted: false,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
const cost = { queries: 0, rows: 0 };
|
|
99
|
+
const prefix = topicPrefix(request.topic);
|
|
100
|
+
const wantedType = request.type?.trim().toLowerCase();
|
|
101
|
+
const extra = [
|
|
102
|
+
...(prefix === undefined ? [] : tokenize(prefix.replace(/\//g, " "))),
|
|
103
|
+
...(wantedType === undefined ? [] : tokenize(wantedType)),
|
|
104
|
+
];
|
|
105
|
+
const considered = new Set();
|
|
106
|
+
/** Applies the filters to one engine hit; returns the page when it survives. */
|
|
107
|
+
const admit = (hit) => {
|
|
108
|
+
considered.add(hit.path);
|
|
109
|
+
const page = catalog.pages.get(hit.path);
|
|
110
|
+
if (page === undefined) {
|
|
111
|
+
removed.unknown.add(hit.path);
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
if (wantedType !== undefined && page.type.toLowerCase() !== wantedType) {
|
|
115
|
+
removed.type.add(hit.path);
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
if (prefix !== undefined && !page.path.startsWith(prefix)) {
|
|
119
|
+
removed.topic.add(hit.path);
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
if (!request.includeStale && isOverdue(page.staleAfter, now)) {
|
|
123
|
+
removed.stale.add(hit.path);
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
return page;
|
|
127
|
+
};
|
|
128
|
+
// First rung: all terms, widening while short and the engine still had more to give.
|
|
129
|
+
let pool = Math.min(limit * POOL_FACTOR, POOL_CAP);
|
|
130
|
+
let first = [];
|
|
131
|
+
for (;;) {
|
|
132
|
+
const { hits: engineHits, exhausted } = await lexComplete(engine, [...terms, ...extra], pool, cost);
|
|
133
|
+
first = [];
|
|
134
|
+
for (const hit of engineHits) {
|
|
135
|
+
const page = admit(hit);
|
|
136
|
+
if (page !== undefined)
|
|
137
|
+
first.push({ page, score: hit.bm25, matched: terms.length });
|
|
138
|
+
}
|
|
139
|
+
if (first.length >= limit || exhausted || pool >= POOL_CAP)
|
|
140
|
+
break;
|
|
141
|
+
pool = Math.min(pool * POOL_FACTOR, POOL_CAP);
|
|
142
|
+
}
|
|
143
|
+
first.sort(order);
|
|
144
|
+
const hits = first
|
|
145
|
+
.slice(0, limit)
|
|
146
|
+
.map((c) => shape(c.page, c.score, "all-terms", now));
|
|
147
|
+
// Relaxed rung: one query per content term, fused by summed BM25, ranked by terms matched then by the sum.
|
|
148
|
+
// The type and topic tokens stay out of these queries: BM25 adds up across terms, so a token present in every
|
|
149
|
+
// candidate would be added once per matched term and move the order within a bucket. The filters still apply.
|
|
150
|
+
// The per-term pool is `limit × 4`, not the widened pool: every row carries its page body, and twelve terms
|
|
151
|
+
// at the cap would materialise thousands of them. A term the engine scores at its frequency floor is in at
|
|
152
|
+
// least half the pages and says nothing about which; it is skipped and named, so the company's own name in
|
|
153
|
+
// every path cannot vote.
|
|
154
|
+
const floored = [];
|
|
155
|
+
if (request.relax !== false && hits.length < limit && terms.length > 1) {
|
|
156
|
+
const relaxedPool = Math.max(1, Math.min(request.relaxedPool ?? limit * POOL_FACTOR, POOL_CAP));
|
|
157
|
+
const taken = new Set(hits.map((h) => h.path));
|
|
158
|
+
const fused = new Map();
|
|
159
|
+
for (const term of terms) {
|
|
160
|
+
const rows = (await lexComplete(engine, [term], relaxedPool, cost)).hits;
|
|
161
|
+
if (rows.length > 0 && rows[0].bm25 < FREQUENCY_FLOOR) {
|
|
162
|
+
floored.push(term);
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
for (const hit of rows) {
|
|
166
|
+
if (taken.has(hit.path))
|
|
167
|
+
continue;
|
|
168
|
+
const page = admit(hit);
|
|
169
|
+
if (page === undefined)
|
|
170
|
+
continue;
|
|
171
|
+
const entry = fused.get(hit.path) ?? { page, score: 0, matched: 0 };
|
|
172
|
+
entry.score += hit.bm25;
|
|
173
|
+
entry.matched += 1;
|
|
174
|
+
fused.set(hit.path, entry);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const best = Math.max(0, ...[...fused.values()].map((e) => e.score));
|
|
178
|
+
const relaxed = [...fused.values()]
|
|
179
|
+
.filter((e) => e.score >= best * RELAXED_FLOOR)
|
|
180
|
+
.sort((a, b) => b.matched - a.matched || order(a, b));
|
|
181
|
+
for (const e of relaxed) {
|
|
182
|
+
if (hits.length >= limit)
|
|
183
|
+
break;
|
|
184
|
+
hits.push({ ...shape(e.page, e.score, "relaxed", now), termsMatched: e.matched });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const strategy = hits[0]?.rung ?? "none";
|
|
188
|
+
return {
|
|
189
|
+
hits,
|
|
190
|
+
strategy,
|
|
191
|
+
terms,
|
|
192
|
+
dropped,
|
|
193
|
+
floored,
|
|
194
|
+
considered: considered.size,
|
|
195
|
+
filteredOut: filteredOut(),
|
|
196
|
+
pool,
|
|
197
|
+
engineQueries: cost.queries,
|
|
198
|
+
rowsFetched: cost.rows,
|
|
199
|
+
topicExhausted: prefix !== undefined && pool >= POOL_CAP && hits.length < limit,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
function shape(page, score, rung, now) {
|
|
203
|
+
const hit = {
|
|
204
|
+
path: page.path,
|
|
205
|
+
title: page.title,
|
|
206
|
+
type: page.type,
|
|
207
|
+
status: page.status,
|
|
208
|
+
trust: page.trust,
|
|
209
|
+
overdue: isOverdue(page.staleAfter, now),
|
|
210
|
+
sources: page.sources.length,
|
|
211
|
+
score,
|
|
212
|
+
rung,
|
|
213
|
+
};
|
|
214
|
+
if (page.resource !== undefined)
|
|
215
|
+
hit.resource = page.resource;
|
|
216
|
+
if (page.description !== undefined)
|
|
217
|
+
hit.description = page.description;
|
|
218
|
+
if (page.staleAfter !== undefined)
|
|
219
|
+
hit.staleAfter = page.staleAfter.raw;
|
|
220
|
+
if (page.replacement !== undefined)
|
|
221
|
+
hit.replacement = page.replacement;
|
|
222
|
+
return hit;
|
|
223
|
+
}
|
|
224
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/search/search.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA2DzD,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,QAAQ,GAAG,GAAG,CAAC;AACrB,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,6HAA6H;AAC7H,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,GAAG,GAAG,IAAI,CAAC;AACjB,MAAM,UAAU,GAA0B;IACxC,gBAAgB,EAAE,CAAC;IACnB,mBAAmB,EAAE,CAAC;IACtB,UAAU,EAAE,CAAC;CACd,CAAC;AAQF,SAAS,WAAW,CAAC,KAAyB;IAC5C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChD,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,CAAY,EAAE,CAAY;IACvC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,GAAG;QAAE,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAChE,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClE,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACpE,CAAC;AAaD;;;;;;;GAOG;AACH,KAAK,UAAU,WAAW,CACxB,MAAc,EACd,KAAwB,EACxB,IAAY,EACZ,IAAe;IAEf,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC;IACnB,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QAClB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAChE,MAAM,GAAG,GAAI,IAAI,CAAC,IAAI,GAAG,CAAC,CAAe,CAAC,IAAI,CAAC;QAC/C,IAAI,GAAG,GAAG,IAAI,CAAC;QACf,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAE,IAAI,CAAC,GAAG,CAAe,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG;YAAE,GAAG,IAAI,CAAC,CAAC;QAC3F,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC7E,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC9D,IAAI,GAAG,GAAG,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC5D,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,WAAW,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,OAAgB,EAChB,MAAc,EACd,OAAsB,EACtB,GAAS;IAET,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;QAC1C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,IAAI,GAAG,EAAY;QACzB,KAAK,EAAE,IAAI,GAAG,EAAY;QAC1B,KAAK,EAAE,IAAI,GAAG,EAAY;QAC1B,OAAO,EAAE,IAAI,GAAG,EAAY;KAC7B,CAAC;IACF,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;QACvB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;QACzB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;QACzB,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI;KAC9B,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,kBAAkB;YAC1B,KAAK;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,WAAW,EAAE;YAC1B,IAAI,EAAE,CAAC;YACP,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,KAAK;SACtB,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG;QACZ,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACrE,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KAC1D,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAY,CAAC;IAEvC,gFAAgF;IAChF,MAAM,KAAK,GAAG,CAAC,GAAc,EAAoB,EAAE;QACjD,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,UAAU,EAAE,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,qFAAqF;IACrF,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,WAAW,EAAE,QAAQ,CAAC,CAAC;IACnD,IAAI,KAAK,GAAgB,EAAE,CAAC;IAC5B,SAAS,CAAC;QACR,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,WAAW,CACvD,MAAM,EACN,CAAC,GAAG,KAAK,EAAE,GAAG,KAAK,CAAC,EACpB,IAAI,EACJ,IAAI,CACL,CAAC;QACF,KAAK,GAAG,EAAE,CAAC;QACX,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,IAAI,KAAK,SAAS;gBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,SAAS,IAAI,IAAI,IAAI,QAAQ;YAAE,MAAM;QAClE,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,WAAW,EAAE,QAAQ,CAAC,CAAC;IAChD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,MAAM,IAAI,GAAgB,KAAK;SAC5B,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAExD,2GAA2G;IAC3G,8GAA8G;IAC9G,8GAA8G;IAC9G,4GAA4G;IAC5G,2GAA2G;IAC3G,2GAA2G;IAC3G,0BAA0B;IAC1B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,IAAI,KAAK,GAAG,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChG,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAK,IAAI,CAAC,CAAC,CAAe,CAAC,IAAI,GAAG,eAAe,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,SAAS;YACX,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAClC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACxB,IAAI,IAAI,KAAK,SAAS;oBAAE,SAAS;gBACjC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;gBACpE,KAAK,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC;gBACxB,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBACnB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;aAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,aAAa,CAAC;aAC9C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;gBAAE,MAAM;YAChC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAkB,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC;IACxD,OAAO;QACL,IAAI;QACJ,QAAQ;QACR,KAAK;QACL,OAAO;QACP,OAAO;QACP,UAAU,EAAE,UAAU,CAAC,IAAI;QAC3B,WAAW,EAAE,WAAW,EAAE;QAC1B,IAAI;QACJ,aAAa,EAAE,IAAI,CAAC,OAAO;QAC3B,WAAW,EAAE,IAAI,CAAC,IAAI;QACtB,cAAc,EAAE,MAAM,KAAK,SAAS,IAAI,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;KAChF,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,IAAU,EAAE,KAAa,EAAE,IAAU,EAAE,GAAS;IAC7D,MAAM,GAAG,GAAc;QACrB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,OAAO,EAAE,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC;QACxC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC5B,KAAK;QACL,IAAI;KACL,CAAC;IACF,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS;QAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC9D,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;QAAE,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACvE,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;QAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACxE,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;QAAE,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IACvE,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["import type { Page, PagePath, Status, Trust } from \"../bundle/model.js\";\nimport { byCodeUnit } from \"../bundle/paths.js\";\nimport type { Catalog } from \"../catalog/model.js\";\nimport { isOverdue } from \"../catalog/provenance.js\";\nimport type { Engine, EngineHit } from \"./engine.js\";\nimport { normaliseQuestion, tokenize } from \"./query.js\";\n\nexport type Rung = \"all-terms\" | \"relaxed\";\n\nexport interface SearchRequest {\n question: string;\n type?: string;\n topic?: string;\n includeStale: boolean;\n limit: number;\n /** Run the relaxed rung when the first rung leaves the answer short (default true); the benchmark turns it off. */\n relax?: boolean;\n /** The per-term pool on the relaxed rung; defaults to the pool the first rung ended with. A tuning knob (D31). */\n relaxedPool?: number;\n}\n\nexport interface SearchHit {\n path: PagePath;\n title: string;\n description?: string;\n type: string;\n status: Status;\n trust: Trust;\n staleAfter?: string;\n overdue: boolean;\n replacement?: PagePath;\n /** How many sources the page lists (intent §6: the count travels on hits). */\n sources: number;\n /** The page's `resource`, when it has one. */\n resource?: string;\n /** Raw BM25. On the all-terms rung it includes the type and topic tokens once; on the relaxed rung it is the sum of the content terms' scores. */\n score: number;\n rung: Rung;\n /** Relaxed rung only: how many informative terms list this page among their best matches (their per-term pool). The first sort key; a term the page holds outside that pool does not count. */\n termsMatched?: number;\n}\n\nexport interface SearchResponse {\n hits: SearchHit[];\n /** The rung that answered: the rung of the first hit; `none` when nothing was found or the question had no content terms. */\n strategy: Rung | \"none\";\n reason?: \"no-content-terms\";\n terms: string[];\n dropped: string[];\n /** Terms the relaxed rung ignored because the engine scores them at its frequency floor (present in at least half the pages). */\n floored: string[];\n /** Distinct engine hits examined across every query. */\n considered: number;\n /** Distinct pages the filters removed, per reason. */\n filteredOut: { type: number; topic: number; stale: number; unknown: number };\n /** The pool size the first rung ended with. */\n pool: number;\n /** Engine queries made and rows they returned, for cost accounting: every row carries its page body. */\n engineQueries: number;\n rowsFetched: number;\n /** A topic filter was set, the pool reached its cap, and the answer is still short: the topic may hold more. */\n topicExhausted: boolean;\n}\n\nconst POOL_FACTOR = 4;\nconst POOL_CAP = 500;\nconst LIMIT_CAP = 25;\nconst RELAXED_FLOOR = 0.01;\n/** Below this, a term's best BM25 is SQLite's floored inverse document frequency: the term is in at least half the pages. */\nconst FREQUENCY_FLOOR = 1e-3;\nconst TIE = 1e-9;\nconst TRUST_RANK: Record<Trust, number> = {\n \"human-reviewed\": 0,\n \"machine-confirmed\": 1,\n unverified: 2,\n};\n\ninterface Candidate {\n page: Page;\n score: number;\n matched: number;\n}\n\nfunction topicPrefix(topic: string | undefined): string | undefined {\n if (topic === undefined) return undefined;\n const trimmed = topic.replace(/^\\/+|\\/+$/g, \"\");\n return trimmed.length === 0 ? undefined : `${trimmed}/`;\n}\n\nfunction order(a: Candidate, b: Candidate): number {\n if (Math.abs(b.score - a.score) > TIE) return b.score - a.score;\n const trust = TRUST_RANK[a.page.trust] - TRUST_RANK[b.page.trust];\n return trust !== 0 ? trust : byCodeUnit(a.page.path, b.page.path);\n}\n\ninterface CompletedPool {\n hits: EngineHit[];\n /** True when the engine had nothing beyond these rows. */\n exhausted: boolean;\n}\n\ninterface QueryCost {\n queries: number;\n rows: number;\n}\n\n/**\n * Asks the engine for `pool` rows and completes the tie group at the cut. qmd orders equal scores by insertion\n * order, which differs from one index build to the next, so a pool that cuts inside a group of equal scores\n * would make the answer depend on the build. One extra row shows whether the cut fell inside a group; when it\n * did, the request widens until a row below the cut's score is seen, the engine runs out, or the cap is\n * reached. Rows below the cut's score are dropped. At the cap the group is cut as the engine cut it, which is\n * the one residual the engine's order can still reach.\n */\nasync function lexComplete(\n engine: Engine,\n terms: readonly string[],\n pool: number,\n cost: QueryCost,\n): Promise<CompletedPool> {\n let ask = pool + 1;\n for (;;) {\n const rows = await engine.lex(terms, ask);\n cost.queries += 1;\n cost.rows += rows.length;\n if (rows.length <= pool) return { hits: rows, exhausted: true };\n const cut = (rows[pool - 1] as EngineHit).bm25;\n let end = pool;\n while (end < rows.length && Math.abs((rows[end] as EngineHit).bm25 - cut) <= TIE) end += 1;\n if (end < rows.length) return { hits: rows.slice(0, end), exhausted: false };\n if (rows.length < ask) return { hits: rows, exhausted: true };\n if (ask > POOL_CAP) return { hits: rows, exhausted: false };\n ask = Math.min(ask * POOL_FACTOR, POOL_CAP + 1);\n }\n}\n\n/**\n * Searches the catalog through the engine. The first rung sends every content term, plus the topic's path\n * segments and the type value, as one query, widening the pool while the filters leave it short and the engine\n * returned a full pool. When still short, the relaxed rung sends one query per content term and fuses by\n * summed BM25, ranked by terms matched. Every engine query completes the tie group at its cut, so the answer\n * does not depend on the engine's order among equal scores. Hits carry provenance and their rung; the response\n * says what was filtered and why.\n */\nexport async function search(\n catalog: Catalog,\n engine: Engine,\n request: SearchRequest,\n now: Date,\n): Promise<SearchResponse> {\n const { terms, dropped } = normaliseQuestion(request.question);\n const limit = Number.isFinite(request.limit)\n ? Math.min(LIMIT_CAP, Math.max(1, Math.floor(request.limit)))\n : 1;\n const removed = {\n type: new Set<PagePath>(),\n topic: new Set<PagePath>(),\n stale: new Set<PagePath>(),\n unknown: new Set<PagePath>(),\n };\n const filteredOut = () => ({\n type: removed.type.size,\n topic: removed.topic.size,\n stale: removed.stale.size,\n unknown: removed.unknown.size,\n });\n if (terms.length === 0) {\n return {\n hits: [],\n strategy: \"none\",\n reason: \"no-content-terms\",\n terms,\n dropped,\n floored: [],\n considered: 0,\n filteredOut: filteredOut(),\n pool: 0,\n engineQueries: 0,\n rowsFetched: 0,\n topicExhausted: false,\n };\n }\n const cost: QueryCost = { queries: 0, rows: 0 };\n const prefix = topicPrefix(request.topic);\n const wantedType = request.type?.trim().toLowerCase();\n const extra = [\n ...(prefix === undefined ? [] : tokenize(prefix.replace(/\\//g, \" \"))),\n ...(wantedType === undefined ? [] : tokenize(wantedType)),\n ];\n const considered = new Set<PagePath>();\n\n /** Applies the filters to one engine hit; returns the page when it survives. */\n const admit = (hit: EngineHit): Page | undefined => {\n considered.add(hit.path);\n const page = catalog.pages.get(hit.path);\n if (page === undefined) {\n removed.unknown.add(hit.path);\n return undefined;\n }\n if (wantedType !== undefined && page.type.toLowerCase() !== wantedType) {\n removed.type.add(hit.path);\n return undefined;\n }\n if (prefix !== undefined && !page.path.startsWith(prefix)) {\n removed.topic.add(hit.path);\n return undefined;\n }\n if (!request.includeStale && isOverdue(page.staleAfter, now)) {\n removed.stale.add(hit.path);\n return undefined;\n }\n return page;\n };\n\n // First rung: all terms, widening while short and the engine still had more to give.\n let pool = Math.min(limit * POOL_FACTOR, POOL_CAP);\n let first: Candidate[] = [];\n for (;;) {\n const { hits: engineHits, exhausted } = await lexComplete(\n engine,\n [...terms, ...extra],\n pool,\n cost,\n );\n first = [];\n for (const hit of engineHits) {\n const page = admit(hit);\n if (page !== undefined) first.push({ page, score: hit.bm25, matched: terms.length });\n }\n if (first.length >= limit || exhausted || pool >= POOL_CAP) break;\n pool = Math.min(pool * POOL_FACTOR, POOL_CAP);\n }\n first.sort(order);\n const hits: SearchHit[] = first\n .slice(0, limit)\n .map((c) => shape(c.page, c.score, \"all-terms\", now));\n\n // Relaxed rung: one query per content term, fused by summed BM25, ranked by terms matched then by the sum.\n // The type and topic tokens stay out of these queries: BM25 adds up across terms, so a token present in every\n // candidate would be added once per matched term and move the order within a bucket. The filters still apply.\n // The per-term pool is `limit × 4`, not the widened pool: every row carries its page body, and twelve terms\n // at the cap would materialise thousands of them. A term the engine scores at its frequency floor is in at\n // least half the pages and says nothing about which; it is skipped and named, so the company's own name in\n // every path cannot vote.\n const floored: string[] = [];\n if (request.relax !== false && hits.length < limit && terms.length > 1) {\n const relaxedPool = Math.max(1, Math.min(request.relaxedPool ?? limit * POOL_FACTOR, POOL_CAP));\n const taken = new Set(hits.map((h) => h.path));\n const fused = new Map<PagePath, Candidate>();\n for (const term of terms) {\n const rows = (await lexComplete(engine, [term], relaxedPool, cost)).hits;\n if (rows.length > 0 && (rows[0] as EngineHit).bm25 < FREQUENCY_FLOOR) {\n floored.push(term);\n continue;\n }\n for (const hit of rows) {\n if (taken.has(hit.path)) continue;\n const page = admit(hit);\n if (page === undefined) continue;\n const entry = fused.get(hit.path) ?? { page, score: 0, matched: 0 };\n entry.score += hit.bm25;\n entry.matched += 1;\n fused.set(hit.path, entry);\n }\n }\n const best = Math.max(0, ...[...fused.values()].map((e) => e.score));\n const relaxed = [...fused.values()]\n .filter((e) => e.score >= best * RELAXED_FLOOR)\n .sort((a, b) => b.matched - a.matched || order(a, b));\n for (const e of relaxed) {\n if (hits.length >= limit) break;\n hits.push({ ...shape(e.page, e.score, \"relaxed\", now), termsMatched: e.matched });\n }\n }\n\n const strategy: Rung | \"none\" = hits[0]?.rung ?? \"none\";\n return {\n hits,\n strategy,\n terms,\n dropped,\n floored,\n considered: considered.size,\n filteredOut: filteredOut(),\n pool,\n engineQueries: cost.queries,\n rowsFetched: cost.rows,\n topicExhausted: prefix !== undefined && pool >= POOL_CAP && hits.length < limit,\n };\n}\n\nfunction shape(page: Page, score: number, rung: Rung, now: Date): SearchHit {\n const hit: SearchHit = {\n path: page.path,\n title: page.title,\n type: page.type,\n status: page.status,\n trust: page.trust,\n overdue: isOverdue(page.staleAfter, now),\n sources: page.sources.length,\n score,\n rung,\n };\n if (page.resource !== undefined) hit.resource = page.resource;\n if (page.description !== undefined) hit.description = page.description;\n if (page.staleAfter !== undefined) hit.staleAfter = page.staleAfter.raw;\n if (page.replacement !== undefined) hit.replacement = page.replacement;\n return hit;\n}\n"]}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
|
2
|
+
const WORD_CHAR = /[\p{L}\p{N}\p{M}]/u;
|
|
3
|
+
/** Where a term matches: at word starts, or anywhere for a term that carries CJK characters. */
|
|
4
|
+
function positions(lower, term) {
|
|
5
|
+
const out = [];
|
|
6
|
+
const anywhere = CJK.test(term);
|
|
7
|
+
let from = 0;
|
|
8
|
+
for (;;) {
|
|
9
|
+
const at = lower.indexOf(term, from);
|
|
10
|
+
if (at === -1)
|
|
11
|
+
break;
|
|
12
|
+
const before = at === 0 ? "" : (lower[at - 1] ?? "");
|
|
13
|
+
if (anywhere || before === "" || !WORD_CHAR.test(before))
|
|
14
|
+
out.push(at);
|
|
15
|
+
from = at + 1;
|
|
16
|
+
}
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
/** Trims a window to word boundaries and marks the cuts. */
|
|
20
|
+
function clip(prose, start, end) {
|
|
21
|
+
let from = start;
|
|
22
|
+
let to = Math.min(end, prose.length);
|
|
23
|
+
// Trim to word boundaries, except inside a CJK run, which has no spaces to trim to.
|
|
24
|
+
const wordy = (c) => WORD_CHAR.test(c) && !CJK.test(c);
|
|
25
|
+
if (from > 0 && wordy(prose[from - 1] ?? "") && wordy(prose[from] ?? "")) {
|
|
26
|
+
while (from < to && wordy(prose[from] ?? ""))
|
|
27
|
+
from++;
|
|
28
|
+
}
|
|
29
|
+
if (to < prose.length && wordy(prose[to] ?? "") && wordy(prose[to - 1] ?? "")) {
|
|
30
|
+
while (to > from && wordy(prose[to - 1] ?? ""))
|
|
31
|
+
to--;
|
|
32
|
+
}
|
|
33
|
+
const text = prose.slice(from, to).trim();
|
|
34
|
+
return `${from > 0 ? "…" : ""}${text}${to < prose.length ? "…" : ""}`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* A snippet of a page for a search hit: the earliest window of `width` characters of the page's stored prose
|
|
38
|
+
* holding the most distinct terms, trimmed to word boundaries and marked where it was cut. When no term matches,
|
|
39
|
+
* the description; failing that, the first characters of the prose; an empty string when there is no prose at all.
|
|
40
|
+
*/
|
|
41
|
+
export function snippet(page, terms, width = 200) {
|
|
42
|
+
const prose = page.prose;
|
|
43
|
+
const fallback = () => {
|
|
44
|
+
const description = page.description ?? "";
|
|
45
|
+
if (description.length > 0)
|
|
46
|
+
return description.length <= width ? description : clip(description, 0, width);
|
|
47
|
+
return prose === undefined ? "" : clip(prose, 0, width);
|
|
48
|
+
};
|
|
49
|
+
if (prose === undefined)
|
|
50
|
+
return fallback();
|
|
51
|
+
if (prose.length <= width)
|
|
52
|
+
return prose;
|
|
53
|
+
// Lower-casing can change a string's length (İ → i̇); when it does, match on the original case instead.
|
|
54
|
+
const lowered = prose.toLowerCase();
|
|
55
|
+
const lower = lowered.length === prose.length ? lowered : prose;
|
|
56
|
+
const matches = [];
|
|
57
|
+
terms.forEach((term, index) => {
|
|
58
|
+
const t = term.toLowerCase();
|
|
59
|
+
if (t.length === 0)
|
|
60
|
+
return;
|
|
61
|
+
for (const at of positions(lower, t))
|
|
62
|
+
matches.push({ at, term: index });
|
|
63
|
+
});
|
|
64
|
+
if (matches.length === 0)
|
|
65
|
+
return fallback();
|
|
66
|
+
matches.sort((a, b) => a.at - b.at);
|
|
67
|
+
// Two pointers over the sorted matches: the window starting at each match, with per-term counts, in linear time.
|
|
68
|
+
const counts = new Map();
|
|
69
|
+
let distinct = 0;
|
|
70
|
+
let right = 0;
|
|
71
|
+
let bestStart = matches[0]?.at ?? 0;
|
|
72
|
+
let bestCount = 0;
|
|
73
|
+
for (let left = 0; left < matches.length; left++) {
|
|
74
|
+
const start = matches[left]?.at ?? 0;
|
|
75
|
+
while (right < matches.length && (matches[right]?.at ?? 0) < start + width) {
|
|
76
|
+
const term = matches[right]?.term ?? 0;
|
|
77
|
+
const seen = counts.get(term) ?? 0;
|
|
78
|
+
if (seen === 0)
|
|
79
|
+
distinct += 1;
|
|
80
|
+
counts.set(term, seen + 1);
|
|
81
|
+
right++;
|
|
82
|
+
}
|
|
83
|
+
if (distinct > bestCount) {
|
|
84
|
+
bestCount = distinct;
|
|
85
|
+
bestStart = start;
|
|
86
|
+
}
|
|
87
|
+
const leaving = matches[left]?.term ?? 0;
|
|
88
|
+
const remaining = (counts.get(leaving) ?? 1) - 1;
|
|
89
|
+
counts.set(leaving, remaining);
|
|
90
|
+
if (remaining === 0)
|
|
91
|
+
distinct -= 1;
|
|
92
|
+
}
|
|
93
|
+
return clip(prose, bestStart, bestStart + width);
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=snippet.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snippet.js","sourceRoot":"","sources":["../../src/search/snippet.ts"],"names":[],"mappings":"AAAA,MAAM,GAAG,GAAG,0EAA0E,CAAC;AACvF,MAAM,SAAS,GAAG,oBAAoB,CAAC;AAEvC,gGAAgG;AAChG,SAAS,SAAS,CAAC,KAAa,EAAE,IAAY;IAC5C,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,SAAS,CAAC;QACR,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,MAAM;QACrB,MAAM,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,QAAQ,IAAI,MAAM,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4DAA4D;AAC5D,SAAS,IAAI,CAAC,KAAa,EAAE,KAAa,EAAE,GAAW;IACrD,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrC,oFAAoF;IACpF,MAAM,KAAK,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxE,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACzE,OAAO,IAAI,GAAG,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAAE,IAAI,EAAE,CAAC;IACvD,CAAC;IACD,IAAI,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9E,OAAO,EAAE,GAAG,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,EAAE,EAAE,CAAC;IACvD,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CACrB,IAA8C,EAC9C,KAAwB,EACxB,KAAK,GAAG,GAAG;IAEX,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,MAAM,QAAQ,GAAG,GAAW,EAAE;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;QAC3C,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YACxB,OAAO,WAAW,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;QACjF,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC,CAAC;IACF,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,EAAE,CAAC;IAC3C,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IACxC,wGAAwG;IACxG,MAAM,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAChE,MAAM,OAAO,GAAwC,EAAE,CAAC;IACxD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC3B,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,EAAE,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACpC,iHAAiH;IACjH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACpC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACrC,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC;YAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,IAAI,KAAK,CAAC;gBAAE,QAAQ,IAAI,CAAC,CAAC;YAC9B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC3B,KAAK,EAAE,CAAC;QACV,CAAC;QACD,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;YACzB,SAAS,GAAG,QAAQ,CAAC;YACrB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAC/B,IAAI,SAAS,KAAK,CAAC;YAAE,QAAQ,IAAI,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,GAAG,KAAK,CAAC,CAAC;AACnD,CAAC","sourcesContent":["const CJK = /[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]/u;\nconst WORD_CHAR = /[\\p{L}\\p{N}\\p{M}]/u;\n\n/** Where a term matches: at word starts, or anywhere for a term that carries CJK characters. */\nfunction positions(lower: string, term: string): number[] {\n const out: number[] = [];\n const anywhere = CJK.test(term);\n let from = 0;\n for (;;) {\n const at = lower.indexOf(term, from);\n if (at === -1) break;\n const before = at === 0 ? \"\" : (lower[at - 1] ?? \"\");\n if (anywhere || before === \"\" || !WORD_CHAR.test(before)) out.push(at);\n from = at + 1;\n }\n return out;\n}\n\n/** Trims a window to word boundaries and marks the cuts. */\nfunction clip(prose: string, start: number, end: number): string {\n let from = start;\n let to = Math.min(end, prose.length);\n // Trim to word boundaries, except inside a CJK run, which has no spaces to trim to.\n const wordy = (c: string): boolean => WORD_CHAR.test(c) && !CJK.test(c);\n if (from > 0 && wordy(prose[from - 1] ?? \"\") && wordy(prose[from] ?? \"\")) {\n while (from < to && wordy(prose[from] ?? \"\")) from++;\n }\n if (to < prose.length && wordy(prose[to] ?? \"\") && wordy(prose[to - 1] ?? \"\")) {\n while (to > from && wordy(prose[to - 1] ?? \"\")) to--;\n }\n const text = prose.slice(from, to).trim();\n return `${from > 0 ? \"…\" : \"\"}${text}${to < prose.length ? \"…\" : \"\"}`;\n}\n\n/**\n * A snippet of a page for a search hit: the earliest window of `width` characters of the page's stored prose\n * holding the most distinct terms, trimmed to word boundaries and marked where it was cut. When no term matches,\n * the description; failing that, the first characters of the prose; an empty string when there is no prose at all.\n */\nexport function snippet(\n page: { prose?: string; description?: string },\n terms: readonly string[],\n width = 200,\n): string {\n const prose = page.prose;\n const fallback = (): string => {\n const description = page.description ?? \"\";\n if (description.length > 0)\n return description.length <= width ? description : clip(description, 0, width);\n return prose === undefined ? \"\" : clip(prose, 0, width);\n };\n if (prose === undefined) return fallback();\n if (prose.length <= width) return prose;\n // Lower-casing can change a string's length (İ → i̇); when it does, match on the original case instead.\n const lowered = prose.toLowerCase();\n const lower = lowered.length === prose.length ? lowered : prose;\n const matches: Array<{ at: number; term: number }> = [];\n terms.forEach((term, index) => {\n const t = term.toLowerCase();\n if (t.length === 0) return;\n for (const at of positions(lower, t)) matches.push({ at, term: index });\n });\n if (matches.length === 0) return fallback();\n matches.sort((a, b) => a.at - b.at);\n // Two pointers over the sorted matches: the window starting at each match, with per-term counts, in linear time.\n const counts = new Map<number, number>();\n let distinct = 0;\n let right = 0;\n let bestStart = matches[0]?.at ?? 0;\n let bestCount = 0;\n for (let left = 0; left < matches.length; left++) {\n const start = matches[left]?.at ?? 0;\n while (right < matches.length && (matches[right]?.at ?? 0) < start + width) {\n const term = matches[right]?.term ?? 0;\n const seen = counts.get(term) ?? 0;\n if (seen === 0) distinct += 1;\n counts.set(term, seen + 1);\n right++;\n }\n if (distinct > bestCount) {\n bestCount = distinct;\n bestStart = start;\n }\n const leaving = matches[left]?.term ?? 0;\n const remaining = (counts.get(leaving) ?? 1) - 1;\n counts.set(leaving, remaining);\n if (remaining === 0) distinct -= 1;\n }\n return clip(prose, bestStart, bestStart + width);\n}\n"]}
|