@jcode.labs/ragmir 6.1.5 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -49
- package/dist/cli.js +194 -24
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +24 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +127 -24
- package/dist/config.js.map +1 -1
- package/dist/context-resources.d.ts.map +1 -1
- package/dist/context-resources.js +19 -8
- package/dist/context-resources.js.map +1 -1
- package/dist/defaults.d.ts +5 -0
- package/dist/defaults.d.ts.map +1 -1
- package/dist/defaults.js +16 -0
- package/dist/defaults.js.map +1 -1
- package/dist/doctor.d.ts.map +1 -1
- package/dist/doctor.js +37 -5
- package/dist/doctor.js.map +1 -1
- package/dist/evaluate.d.ts.map +1 -1
- package/dist/evaluate.js +11 -6
- package/dist/evaluate.js.map +1 -1
- package/dist/files.d.ts +27 -1
- package/dist/files.d.ts.map +1 -1
- package/dist/files.js +344 -100
- package/dist/files.js.map +1 -1
- package/dist/freshness.d.ts +16 -0
- package/dist/freshness.d.ts.map +1 -0
- package/dist/freshness.js +97 -0
- package/dist/freshness.js.map +1 -0
- package/dist/generation-retention.d.ts +12 -0
- package/dist/generation-retention.d.ts.map +1 -1
- package/dist/generation-retention.js +13 -2
- package/dist/generation-retention.js.map +1 -1
- package/dist/git-hooks.d.ts +25 -0
- package/dist/git-hooks.d.ts.map +1 -0
- package/dist/git-hooks.js +173 -0
- package/dist/git-hooks.js.map +1 -0
- package/dist/index-diagnostics.d.ts +9 -1
- package/dist/index-diagnostics.d.ts.map +1 -1
- package/dist/index-diagnostics.js +35 -0
- package/dist/index-diagnostics.js.map +1 -1
- package/dist/index-policy.d.ts.map +1 -1
- package/dist/index-policy.js +4 -0
- package/dist/index-policy.js.map +1 -1
- package/dist/ingest.d.ts +6 -0
- package/dist/ingest.d.ts.map +1 -1
- package/dist/ingest.js +44 -21
- package/dist/ingest.js.map +1 -1
- package/dist/init.d.ts.map +1 -1
- package/dist/init.js +19 -4
- package/dist/init.js.map +1 -1
- package/dist/knowledge-bases.d.ts +18 -1
- package/dist/knowledge-bases.d.ts.map +1 -1
- package/dist/knowledge-bases.js +73 -4
- package/dist/knowledge-bases.js.map +1 -1
- package/dist/mcp-output.d.ts +14 -3
- package/dist/mcp-output.d.ts.map +1 -1
- package/dist/mcp-output.js +37 -0
- package/dist/mcp-output.js.map +1 -1
- package/dist/mcp-registrations.d.ts +9 -0
- package/dist/mcp-registrations.d.ts.map +1 -0
- package/dist/mcp-registrations.js +247 -0
- package/dist/mcp-registrations.js.map +1 -0
- package/dist/mcp-scope.d.ts +32 -0
- package/dist/mcp-scope.d.ts.map +1 -0
- package/dist/mcp-scope.js +114 -0
- package/dist/mcp-scope.js.map +1 -0
- package/dist/mcp-session.d.ts +37 -0
- package/dist/mcp-session.d.ts.map +1 -0
- package/dist/mcp-session.js +53 -0
- package/dist/mcp-session.js.map +1 -0
- package/dist/mcp-summaries.d.ts +5 -0
- package/dist/mcp-summaries.d.ts.map +1 -1
- package/dist/mcp-summaries.js +30 -0
- package/dist/mcp-summaries.js.map +1 -1
- package/dist/mcp.d.ts +6 -0
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +410 -103
- package/dist/mcp.js.map +1 -1
- package/dist/ocr.d.ts +51 -0
- package/dist/ocr.d.ts.map +1 -1
- package/dist/ocr.js +210 -4
- package/dist/ocr.js.map +1 -1
- package/dist/package-manager.d.ts.map +1 -1
- package/dist/package-manager.js +2 -1
- package/dist/package-manager.js.map +1 -1
- package/dist/preview.d.ts.map +1 -1
- package/dist/preview.js +3 -7
- package/dist/preview.js.map +1 -1
- package/dist/quality-report.d.ts.map +1 -1
- package/dist/quality-report.js +1 -1
- package/dist/quality-report.js.map +1 -1
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +254 -40
- package/dist/query.js.map +1 -1
- package/dist/ranking.d.ts +56 -6
- package/dist/ranking.d.ts.map +1 -1
- package/dist/ranking.js +187 -16
- package/dist/ranking.js.map +1 -1
- package/dist/retrieval-diversity.d.ts +5 -0
- package/dist/retrieval-diversity.d.ts.map +1 -1
- package/dist/retrieval-diversity.js +4 -2
- package/dist/retrieval-diversity.js.map +1 -1
- package/dist/search-output.d.ts.map +1 -1
- package/dist/search-output.js +2 -0
- package/dist/search-output.js.map +1 -1
- package/dist/security.js +1 -1
- package/dist/security.js.map +1 -1
- package/dist/setup-prompt.d.ts +1 -1
- package/dist/setup-prompt.d.ts.map +1 -1
- package/dist/setup-prompt.js +3 -3
- package/dist/setup.js +1 -1
- package/dist/setup.js.map +1 -1
- package/dist/skill.d.ts +2 -0
- package/dist/skill.d.ts.map +1 -1
- package/dist/skill.js +122 -14
- package/dist/skill.js.map +1 -1
- package/dist/source-paths.d.ts +36 -0
- package/dist/source-paths.d.ts.map +1 -0
- package/dist/source-paths.js +157 -0
- package/dist/source-paths.js.map +1 -0
- package/dist/source-weights.d.ts +28 -0
- package/dist/source-weights.d.ts.map +1 -0
- package/dist/source-weights.js +241 -0
- package/dist/source-weights.js.map +1 -0
- package/dist/storage-maintenance.d.ts +3 -1
- package/dist/storage-maintenance.d.ts.map +1 -1
- package/dist/storage-maintenance.js +21 -2
- package/dist/storage-maintenance.js.map +1 -1
- package/dist/storage-usage.d.ts +21 -0
- package/dist/storage-usage.d.ts.map +1 -0
- package/dist/storage-usage.js +118 -0
- package/dist/storage-usage.js.map +1 -0
- package/dist/types.d.ts +127 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -1
package/dist/preview.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"preview.js","sourceRoot":"","sources":["../src/preview.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"preview.js","sourceRoot":"","sources":["../src/preview.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAA;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAUxE,MAAM,iBAAiB,GAAG,CAAC,CAAA;AAC3B,MAAM,2BAA2B,GAAG,CAAC,CAAA;AACrC,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,2BAA2B,GAAG,EAAE,CAAA;AAEtC,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAO,GAAyB,EAAE;IACpE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,EAAE,qBAAqB,EAAE,KAAK,EAAE,CAAC,CAAA;IACtF,MAAM,cAAc,GAAG,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,sBAAsB,CACrC,OAAO,CAAC,QAAQ,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,UAAU,CACX,CAAA;IACD,MAAM,gBAAgB,GAAG,sBAAsB,CAC7C,OAAO,CAAC,gBAAgB,EACxB,2BAA2B,EAC3B,2BAA2B,EAC3B,kBAAkB,CACnB,CAAA;IACD,MAAM,aAAa,GAAG,SAAS,CAAC,cAAc;SAC3C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;SACrF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAA;IAClF,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IACtD,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAC1C,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAC7F,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC,CACzE,CAAA;IACD,MAAM,KAAK,GAAkB,EAAE,CAAA;IAC/B,MAAM,MAAM,GAA4B,EAAE,CAAA;IAC1C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC5B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,cAAc;QACd,cAAc;QACd,YAAY,EAAE,aAAa,CAAC,MAAM;QAClC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QACtE,KAAK;QACL,MAAM;KACP,CAAA;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,IAAgB,EAChB,MAA8C,EAC9C,gBAAwB;IAExB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;QAC3E,OAAO;YACL,IAAI,EAAE;gBACJ,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;gBAC/B,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,IAAI;gBACvB,UAAU,EAAE,mBAAmB,CAAC,MAAM,CAAC;gBACvC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC;gBAC3D,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,gBAAgB,CAAC;aAC7D;SACF,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,IAAI,CAAC,YAAY;gBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAChE;SACF,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAgB;IACpC,OAAO;QACL,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;KAC/B,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,sBAAsB,CAAC,KAAK,CAAC,CAAA;AACtC,CAAC;AAED,SAAS,cAAc,CAAC,IAAgB,EAAE,QAAkB;IAC1D,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,WAAW,CAAC,YAAoB,EAAE,MAAc;IACvD,OAAO,YAAY,KAAK,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAA;AACzE,CAAC;AAED,SAAS,qBAAqB,CAAC,QAA8B;IAC3D,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAC9E,CAAC;AAED,SAAS,sBAAsB,CAC7B,KAAyB,EACzB,QAAgB,EAChB,OAAe,EACf,IAAY;IAEZ,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAA;IACjB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,8BAA8B,CAAC,CAAA;IACxD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;AACjC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality-report.d.ts","sourceRoot":"","sources":["../src/quality-report.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAEnB,MAAM,YAAY,CAAA;AAEnB,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,aAAa,GAAG,MAAM,CAqBxE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG;IAC1E,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB,EAAE,MAAM,CAAA;CAC1B,CAcA;AAED,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAE7F;AAED,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,0BAA0B,CAAC,GAC3D,MAAM,CAGR;AAED,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,kBAAkB,GAAG,SAAS,EACtC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"quality-report.d.ts","sourceRoot":"","sources":["../src/quality-report.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,kBAAkB,EAEnB,MAAM,YAAY,CAAA;AAEnB,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,aAAa,GAAG,MAAM,CAqBxE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG;IAC1E,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB,EAAE,MAAM,CAAA;CAC1B,CAcA;AAED,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAE7F;AAED,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,0BAA0B,CAAC,GAC3D,MAAM,CAGR;AAED,wBAAsB,yBAAyB,CAC7C,MAAM,EAAE,kBAAkB,GAAG,SAAS,EACtC,QAAQ,EAAE,aAAa,GAAG,IAAI,EAC9B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC,CA6ClB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,kBAAkB,CAyBhF"}
|
package/dist/quality-report.js
CHANGED
|
@@ -57,7 +57,7 @@ export async function isCompatibleQualityReport(report, manifest, config) {
|
|
|
57
57
|
report.embeddingModelDigest !== config.embeddingModelDigest ||
|
|
58
58
|
report.retrievalProfile !== config.retrievalProfile ||
|
|
59
59
|
report.rankingPolicyFingerprint !==
|
|
60
|
-
rankingPolicyFingerprint(rankingPolicyFor(config.embeddingProvider, config.retrievalProfile, config.maxChunksPerDocument))) {
|
|
60
|
+
rankingPolicyFingerprint(rankingPolicyFor(config.embeddingProvider, config.retrievalProfile, config.maxChunksPerDocument, config.embeddingModel))) {
|
|
61
61
|
return false;
|
|
62
62
|
}
|
|
63
63
|
const goldenPath = path.resolve(config.projectRoot, report.goldenPath);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"quality-report.js","sourceRoot":"","sources":["../src/quality-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AASzE,MAAM,UAAU,wBAAwB,CAAC,QAAuB;IAC9D,OAAO,MAAM,CACX,UAAU,CAAC;QACT,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,IAAI,IAAI;QAC/D,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB,IAAI,IAAI;QAC3D,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,IAAI,IAAI;QAC/D,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,IAAI;QACjD,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB,IAAI,IAAI;QAC3D,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,IAAI;QACrC,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE;QACzC,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,EAAE;KACtC,CAAC,CACH,CAAA;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAkC;IAKvE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACnC,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,MAAM,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAA;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,SAAS,IAAI,CAAC,CAAA;QACd,UAAU,IAAI,IAAI,CAAC,UAAU,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACjE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB,EAAE,MAAoB;IAC1E,OAAO,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,MAA4D;IAE5D,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,MAAM,CAAA;IACzD,OAAO,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAsC,EACtC,QAA8B,EAC9B,MAAc;IAEd,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IACE,MAAM,CAAC,gBAAgB,KAAK,wBAAwB,CAAC,QAAQ,CAAC;QAC9D,MAAM,CAAC,sBAAsB,KAAK,QAAQ,CAAC,sBAAsB;QACjE,MAAM,CAAC,iBAAiB,KAAK,MAAM,CAAC,iBAAiB;QACrD,MAAM,CAAC,cAAc,KAAK,MAAM,CAAC,cAAc;QAC/C,MAAM,CAAC,sBAAsB,KAAK,MAAM,CAAC,sBAAsB;QAC/D,MAAM,CAAC,oBAAoB,KAAK,MAAM,CAAC,oBAAoB;QAC3D,MAAM,CAAC,gBAAgB,KAAK,MAAM,CAAC,gBAAgB;QACnD,MAAM,CAAC,wBAAwB;YAC7B,wBAAwB,CACtB,gBAAgB,CACd,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,gBAAgB,EACvB,MAAM,CAAC,oBAAoB,
|
|
1
|
+
{"version":3,"file":"quality-report.js","sourceRoot":"","sources":["../src/quality-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,wBAAwB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AASzE,MAAM,UAAU,wBAAwB,CAAC,QAAuB;IAC9D,OAAO,MAAM,CACX,UAAU,CAAC;QACT,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,IAAI,IAAI;QAC/D,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB,IAAI,IAAI;QAC3D,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB,IAAI,IAAI;QAC/D,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,IAAI;QACjD,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB,IAAI,IAAI;QAC3D,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS,IAAI,IAAI;QACrC,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,EAAE;QACzC,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,EAAE;KACtC,CAAC,CACH,CAAA;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAkC;IAKvE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACnC,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,MAAM,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAA;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,SAAS,IAAI,CAAC,CAAA;QACd,UAAU,IAAI,IAAI,CAAC,UAAU,CAAA;QAC7B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QACjE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;AAC3E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB,EAAE,MAAoB;IAC1E,OAAO,MAAM,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,MAA4D;IAE5D,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,MAAM,CAAA;IACzD,OAAO,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAsC,EACtC,QAA8B,EAC9B,MAAc;IAEd,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IACE,MAAM,CAAC,gBAAgB,KAAK,wBAAwB,CAAC,QAAQ,CAAC;QAC9D,MAAM,CAAC,sBAAsB,KAAK,QAAQ,CAAC,sBAAsB;QACjE,MAAM,CAAC,iBAAiB,KAAK,MAAM,CAAC,iBAAiB;QACrD,MAAM,CAAC,cAAc,KAAK,MAAM,CAAC,cAAc;QAC/C,MAAM,CAAC,sBAAsB,KAAK,MAAM,CAAC,sBAAsB;QAC/D,MAAM,CAAC,oBAAoB,KAAK,MAAM,CAAC,oBAAoB;QAC3D,MAAM,CAAC,gBAAgB,KAAK,MAAM,CAAC,gBAAgB;QACnD,MAAM,CAAC,wBAAwB;YAC7B,wBAAwB,CACtB,gBAAgB,CACd,MAAM,CAAC,iBAAiB,EACxB,MAAM,CAAC,gBAAgB,EACvB,MAAM,CAAC,oBAAoB,EAC3B,MAAM,CAAC,cAAc,CACtB,CACF,EACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;IACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;IACxE,IACE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QAClC,kBAAkB,KAAK,IAAI;QAC3B,kBAAkB,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,EAC9C,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,kBAAkB,GAAG,MAAM,eAAe,CAAC,UAAU,CAAC,CAAA;QAC5D,MAAM,EAAE,wBAAwB,EAAE,YAAY,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,CAAA;QAC5E,OAAO,CACL,kBAAkB,KAAK,MAAM,CAAC,iBAAiB;YAC/C,wBAAwB,CAAC,cAAc,CAAC,KAAK,MAAM,CAAC,wBAAwB,CAC7E,CAAA;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAc;IACjD,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,aAAa,KAAK,CAAC;QACzB,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QACpC,OAAO,KAAK,CAAC,iBAAiB,KAAK,QAAQ;QAC3C,OAAO,KAAK,CAAC,gBAAgB,KAAK,QAAQ;QAC1C,OAAO,KAAK,CAAC,sBAAsB,KAAK,QAAQ;QAChD,CAAC,KAAK,CAAC,iBAAiB,KAAK,YAAY,IAAI,KAAK,CAAC,iBAAiB,KAAK,cAAc,CAAC;QACxF,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;QACxC,OAAO,KAAK,CAAC,sBAAsB,KAAK,QAAQ;QAChD,CAAC,KAAK,CAAC,oBAAoB,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,oBAAoB,KAAK,QAAQ,CAAC;QACvF,CAAC,KAAK,CAAC,gBAAgB,KAAK,MAAM;YAChC,KAAK,CAAC,gBAAgB,KAAK,UAAU;YACrC,KAAK,CAAC,gBAAgB,KAAK,SAAS;YACpC,KAAK,CAAC,gBAAgB,KAAK,QAAQ,CAAC;QACtC,OAAO,KAAK,CAAC,wBAAwB,KAAK,QAAQ;QAClD,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ;QAC/B,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC;QAC/B,oBAAoB,CAAC,KAAK,CAAC,UAAU,CAAC;QACtC,KAAK,CAAC,MAAM,KAAK,IAAI;QACrB,KAAK,CAAC,oBAAoB,KAAK,IAAI;QACnC,OAAO,KAAK,CAAC,wBAAwB,KAAK,QAAQ,CACnD,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC;QAC9B,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC;QAChC,YAAY,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAC1C,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5B,YAAY,CAAC,KAAK,CAAC,iBAAiB,CAAC;QACrC,YAAY,CAAC,KAAK,CAAC,iBAAiB,CAAC,CACtC,CAAA;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC;QAC7B,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC;QAC9B,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC;QAChC,YAAY,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAC1C,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5B,YAAY,CAAC,KAAK,CAAC,iBAAiB,CAAC;QACrC,YAAY,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAC7C,CAAA;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AACxF,CAAC;AAED,SAAS,MAAM,CAAC,KAA0B;IACxC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IAC/C,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;aAC7B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;aACpD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;aACpE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAA;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAA;AACxC,CAAC"}
|
package/dist/query.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../src/query.ts"],"names":[],"mappings":"AAIA,OAAO,EAEL,KAAK,UAAU,EAMhB,MAAM,kBAAkB,CAAA;AAwCzB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAGnD,OAAO,KAAK,EACV,MAAM,EAEN,qBAAqB,EACrB,gBAAgB,EAEhB,gBAAgB,EAEhB,aAAa,EACb,YAAY,EAGb,MAAM,YAAY,CAAA;AA0DnB,eAAO,MAAM,qCAAqC,6BAA6B,CAAA;AAsD/E,wBAAsB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAIhG;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,UAAU,EACvB,YAAY,CAAC,EAAE,WAAW,EAC1B,gBAAgB,CAAC,EAAE,iBAAiB,EACpC,mBAAmB,UAAQ,GAC1B,OAAO,CAAC,YAAY,EAAE,CAAC,CAkBzB;AAkWD,wBAAgB,oBAAoB,CAClC,eAAe,EAAE,MAAM,EACvB,OAAO,GAAE,gBAA6B,GACrC,MAAM,CAER;AAED,wBAAgB,qBAAqB,CACnC,eAAe,EAAE,MAAM,EACvB,OAAO,GAAE,gBAA6B,GACrC,MAAM,CAER;AAMD,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAG3B;AAED,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,qBAAqB,EAC9B,MAAM,EAAE,MAAM,EACd,UAAU,CAAC,EAAE,UAAU,EACvB,gBAAgB,CAAC,EAAE,iBAAiB,GACnC,OAAO,CAAC,gBAAgB,CAAC,CAe3B"}
|
package/dist/query.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { channel } from "node:diagnostics_channel";
|
|
3
|
+
import { stat } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
3
5
|
import { BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery, } from "@lancedb/lancedb";
|
|
4
6
|
import { citationForCoordinates, stripCitationCoordinates } from "./citation.js";
|
|
5
7
|
import { loadConfig } from "./config.js";
|
|
@@ -8,12 +10,15 @@ import { embedText } from "./embeddings.js";
|
|
|
8
10
|
import { RagmirError } from "./errors.js";
|
|
9
11
|
import { acquireGenerationReadLease } from "./generation-retention.js";
|
|
10
12
|
import { indexFreshnessWarning } from "./index-diagnostics.js";
|
|
13
|
+
import { lexicalPolicyCurrent } from "./index-policy.js";
|
|
11
14
|
import { knowledgeBaseIdentity } from "./knowledge-bases.js";
|
|
12
15
|
import { addLexicalDocument, lexicalDocument, lexicalDocumentScore, } from "./lexical-scoring.js";
|
|
13
16
|
import { operationSignal, throwIfAborted } from "./operation.js";
|
|
14
17
|
import { sanitizeRetrievalQuery } from "./query-sanitizer.js";
|
|
15
|
-
import { abstentionFilter, fusionCandidateDepth, queryEvidence, rankHybridRows, rankingPolicyFingerprint, rankingPolicyFor, tokensAreLexicallyRelated, } from "./ranking.js";
|
|
18
|
+
import { abstentionFilter, fusionCandidateDepth, fuzzyMatchPrefilter, queryEvidence, queryTermCoverage, rankHybridRows, rankingPolicyFingerprint, rankingPolicyFor, tokensAreFuzzyRelated, tokensAreLexicallyRelated, weakEvidenceFilter, } from "./ranking.js";
|
|
16
19
|
import { selectDiverseRows } from "./retrieval-diversity.js";
|
|
20
|
+
import { canonicalUserPath, isCaseInsensitiveFileSystem, resolveExistingPath, unicodePathVariants, } from "./source-paths.js";
|
|
21
|
+
import { definitionBoostOf, localeDocumentKey, sourceWeightOf } from "./source-weights.js";
|
|
17
22
|
import { closeIndexReadSnapshot, loadIndexReadSnapshot } from "./store.js";
|
|
18
23
|
import { normalizeForMatch, tokenize } from "./text.js";
|
|
19
24
|
import { configureAdaptiveVectorQuery, vectorIndexManifestCompatible } from "./vector-index.js";
|
|
@@ -31,6 +36,14 @@ const DIVERSITY_CANDIDATE_MULTIPLIER = 4;
|
|
|
31
36
|
const MAX_VECTOR_CANDIDATES = 1_000;
|
|
32
37
|
const MAX_LEXICAL_CANDIDATES = 4_000;
|
|
33
38
|
const FULL_TEXT_INDEX_NAME = "searchText_idx";
|
|
39
|
+
// Document frequencies only weigh words: past 2% of the passages a word is common either way, and
|
|
40
|
+
// counting deeper only adds full-text reads (3 to 6 ms per word on a 44,000-passage index).
|
|
41
|
+
const TERM_FREQUENCY_SHARE = 0.02;
|
|
42
|
+
const MIN_TERM_FREQUENCY_LIMIT = 500;
|
|
43
|
+
const MAX_TERM_FREQUENCY_LIMIT = 5_000;
|
|
44
|
+
const MAX_CACHED_TERM_GENERATIONS = 4;
|
|
45
|
+
const UNSEGMENTED_SCRIPT_PATTERN = /[\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
|
46
|
+
const termFrequencyCache = new Map();
|
|
34
47
|
const LEXICAL_IDENTIFIER_PATTERN = /[\p{L}\p{N}]+(?:[-_.][\p{L}\p{N}]+)+/gu;
|
|
35
48
|
const SOURCE_PATH_QUERY_PATTERN = /^[\p{L}\p{N}_. /\\-]+\.[a-z0-9]{1,12}$/iu;
|
|
36
49
|
export const QUERY_EXPLANATION_DIAGNOSTICS_CHANNEL = "ragmir:query-explanation";
|
|
@@ -57,6 +70,8 @@ const SEARCH_COLUMNS = [
|
|
|
57
70
|
"locationLabel",
|
|
58
71
|
"cellStart",
|
|
59
72
|
"cellEnd",
|
|
73
|
+
"bytes",
|
|
74
|
+
"mtimeMs",
|
|
60
75
|
];
|
|
61
76
|
const VECTOR_SEARCH_COLUMNS = [...SEARCH_COLUMNS, "_distance"];
|
|
62
77
|
const FTS_SEARCH_COLUMNS = [...SEARCH_COLUMNS, "_score"];
|
|
@@ -105,8 +120,13 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
105
120
|
if (!sanitized.query || queryTokens.length === 0) {
|
|
106
121
|
return [];
|
|
107
122
|
}
|
|
108
|
-
const
|
|
123
|
+
const scope = resolveSearchScope(config, options.scope);
|
|
124
|
+
const retrievalPredicate = searchPredicate([scope?.includePaths, options.includePaths], [...(scope?.excludePaths ?? []), ...(options.excludePaths ?? [])], options.contextPaths);
|
|
109
125
|
let embeddingQueueMs = 0;
|
|
126
|
+
// Indexes built before folded keyword text cannot answer exact word frequencies reliably.
|
|
127
|
+
const termFrequenciesPromise = snapshot.manifest && lexicalPolicyCurrent(snapshot.manifest)
|
|
128
|
+
? queryTermFrequencies(table, queryTokens, snapshot).catch(() => null)
|
|
129
|
+
: Promise.resolve(null);
|
|
110
130
|
const lexicalCandidatesPromise = lexicalCandidateRows(table, sanitized.query, {
|
|
111
131
|
ftsLimit: lexicalCandidateLimit(candidateDemand, config.retrievalProfile),
|
|
112
132
|
fallbackBatchSize: config.hybridTextScanLimit,
|
|
@@ -120,10 +140,25 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
120
140
|
}).then(async (vector) => {
|
|
121
141
|
throwIfAborted(signal);
|
|
122
142
|
const manifest = assertVectorIndexCompatibility(config, snapshot.manifest, vector.length);
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
.
|
|
126
|
-
.
|
|
143
|
+
const limit = vectorCandidateLimit(candidateDemand, config.retrievalProfile);
|
|
144
|
+
const nearest = async (predicate) => {
|
|
145
|
+
const vectorQuery = configureAdaptiveVectorQuery(table.vectorSearch(vector).select(VECTOR_SEARCH_COLUMNS), manifest.vectorIndex, options.vectorSearchMode === "exact");
|
|
146
|
+
return (await (predicate ? vectorQuery.where(predicate) : vectorQuery)
|
|
147
|
+
.limit(limit)
|
|
148
|
+
.toArray());
|
|
149
|
+
};
|
|
150
|
+
const rows = await nearest(retrievalPredicate);
|
|
151
|
+
const crowding = crowdingDocuments(rows, topK);
|
|
152
|
+
if (crowding.length === 0) {
|
|
153
|
+
return rows;
|
|
154
|
+
}
|
|
155
|
+
// One large file (an export, a log, a spreadsheet) filled the pool with its own chunks, yet
|
|
156
|
+
// shows at most maxChunksPerDocument of them; a second pass reaches the other documents.
|
|
157
|
+
throwIfAborted(signal);
|
|
158
|
+
const others = `NOT (${crowding.map(relativePathEquals).join(" OR ")})`;
|
|
159
|
+
const more = await nearest(retrievalPredicate ? `(${retrievalPredicate}) AND ${others}` : others);
|
|
160
|
+
const seen = new Set(rows.map(rowKey));
|
|
161
|
+
return [...rows, ...more.filter((row) => !seen.has(rowKey(row)))];
|
|
127
162
|
});
|
|
128
163
|
const [vectorRowsOutcome, lexicalCandidatesOutcome] = await Promise.allSettled([
|
|
129
164
|
vectorRowsPromise,
|
|
@@ -137,24 +172,39 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
137
172
|
}
|
|
138
173
|
const vectorRows = vectorRowsOutcome.value;
|
|
139
174
|
const lexicalCandidates = lexicalCandidatesOutcome.value;
|
|
175
|
+
const termStatistics = withFuzzyMatches(await termFrequenciesPromise, queryTokens, [
|
|
176
|
+
...lexicalCandidates.rows,
|
|
177
|
+
...vectorRows,
|
|
178
|
+
]);
|
|
140
179
|
throwIfAborted(signal);
|
|
141
|
-
const rankingPolicy = rankingPolicyFor(config.embeddingProvider, config.retrievalProfile, maxChunksPerDocument);
|
|
142
|
-
const
|
|
180
|
+
const rankingPolicy = rankingPolicyFor(config.embeddingProvider, config.retrievalProfile, maxChunksPerDocument, config.embeddingModel);
|
|
181
|
+
const weightOf = sourceWeightOf(config);
|
|
182
|
+
const definitionBoost = definitionBoostOf(queryTokens);
|
|
183
|
+
const rankedRows = rankHybridRows(sanitized.query, vectorRows, lexicalCandidates.rows, rankingPolicy, (row) => weightOf(row.relativePath) * definitionBoost(row));
|
|
143
184
|
const isExactPathMatch = (row) => lexicalCandidates.exactPathMatches.has(rowKey(row));
|
|
144
|
-
const passesAbstention = abstentionFilter(evidence, rankingPolicy);
|
|
185
|
+
const passesAbstention = abstentionFilter(evidence, rankingPolicy, termStatistics);
|
|
186
|
+
const hasWeakEvidence = weakEvidenceFilter(evidence, rankingPolicy, termStatistics);
|
|
187
|
+
const coverageOf = options.explain && termStatistics ? queryTermCoverage(evidence, termStatistics) : null;
|
|
145
188
|
const orderedRows = rankedRows.sort((left, right) => Number(isExactPathMatch(right.row)) - Number(isExactPathMatch(left.row)));
|
|
146
189
|
const diversity = selectDiverseRows(orderedRows, {
|
|
147
190
|
topK,
|
|
148
191
|
maxChunksPerDocument,
|
|
149
192
|
isExactPathMatch,
|
|
193
|
+
...(config.collapseLocaleVariants
|
|
194
|
+
? { documentKeyOf: (row) => localeDocumentKey(row.relativePath) }
|
|
195
|
+
: {}),
|
|
150
196
|
isRelevant: (row) => isExactPathMatch(row) || passesAbstention(row),
|
|
151
197
|
});
|
|
152
198
|
const rows = diversity.rows;
|
|
153
|
-
const contextByRow = await
|
|
199
|
+
const [contextByRow, sourceStatuses] = await Promise.all([
|
|
200
|
+
contextChunksByRow(table, rows, contextRadius, retrievalPredicate),
|
|
201
|
+
sourceStatusesForRows(config.projectRoot, rows),
|
|
202
|
+
]);
|
|
154
203
|
throwIfAborted(signal);
|
|
155
204
|
const knowledgeBaseId = knowledgeBaseIdentity(config.projectRoot)?.id ?? null;
|
|
156
205
|
const results = rows.map((row) => {
|
|
157
206
|
const vectorDistance = typeof row.row._distance === "number" ? row.row._distance : null;
|
|
207
|
+
const sourceStatus = sourceStatuses.get(row.row.relativePath);
|
|
158
208
|
return {
|
|
159
209
|
source: row.row.source,
|
|
160
210
|
relativePath: row.row.relativePath,
|
|
@@ -171,6 +221,10 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
171
221
|
pageEnd: nullablePageNumber(row.row.pageEnd),
|
|
172
222
|
context: contextByRow.get(rowKey(row.row)) ?? [],
|
|
173
223
|
evidence: evidenceVersionForRow(row.row, snapshot, knowledgeBaseId),
|
|
224
|
+
...(sourceStatus === undefined ? {} : { sourceStatus }),
|
|
225
|
+
...(!isExactPathMatch(row.row) && hasWeakEvidence(row.row)
|
|
226
|
+
? { weakEvidence: true }
|
|
227
|
+
: {}),
|
|
174
228
|
...(options.explain
|
|
175
229
|
? {
|
|
176
230
|
score: {
|
|
@@ -179,6 +233,7 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
179
233
|
originalQueryLength: sanitized.originalLength,
|
|
180
234
|
fusion: "rrf",
|
|
181
235
|
combinedScore: row.combinedScore,
|
|
236
|
+
scoreMultiplier: row.scoreMultiplier,
|
|
182
237
|
vectorContribution: row.vectorScore,
|
|
183
238
|
lexicalContribution: row.lexicalScore,
|
|
184
239
|
vectorRank: row.vectorRank,
|
|
@@ -202,6 +257,7 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
202
257
|
workloadQueueMs: workloadQueueMs + embeddingQueueMs,
|
|
203
258
|
rankingPolicyFingerprint: rankingPolicyFingerprint(rankingPolicy),
|
|
204
259
|
matchedTerms: matchedQueryTerms(config.projectRoot, queryTokens, row.row.searchText),
|
|
260
|
+
queryTermCoverage: coverageOf ? coverageOf(row.row.searchText) : null,
|
|
205
261
|
},
|
|
206
262
|
}
|
|
207
263
|
: {}),
|
|
@@ -209,6 +265,132 @@ async function searchWithinGeneration(query, options, config, snapshot, activeSi
|
|
|
209
265
|
});
|
|
210
266
|
return results;
|
|
211
267
|
}
|
|
268
|
+
const CROWDING_MIN_CHUNKS = 4;
|
|
269
|
+
const CROWDING_SHARE = 0.25;
|
|
270
|
+
/** Documents holding a quarter or more of a candidate pool that reaches too few documents. */
|
|
271
|
+
function crowdingDocuments(rows, topK) {
|
|
272
|
+
if (rows.length < CROWDING_MIN_CHUNKS * 2) {
|
|
273
|
+
return [];
|
|
274
|
+
}
|
|
275
|
+
const counts = new Map();
|
|
276
|
+
for (const row of rows) {
|
|
277
|
+
counts.set(row.relativePath, (counts.get(row.relativePath) ?? 0) + 1);
|
|
278
|
+
}
|
|
279
|
+
if (counts.size >= Math.min(rows.length, topK * 3)) {
|
|
280
|
+
return [];
|
|
281
|
+
}
|
|
282
|
+
const threshold = Math.max(CROWDING_MIN_CHUNKS, Math.ceil(rows.length * CROWDING_SHARE));
|
|
283
|
+
return [...counts.entries()]
|
|
284
|
+
.filter(([, count]) => count >= threshold)
|
|
285
|
+
.map(([relativePath]) => relativePath);
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Compare each cited file with the size and modification time recorded at indexing. Agents can then
|
|
289
|
+
* tell evidence that no longer matches the file on disk from current evidence, at one stat per file.
|
|
290
|
+
*/
|
|
291
|
+
async function sourceStatusesForRows(projectRoot, rows) {
|
|
292
|
+
const byPath = new Map();
|
|
293
|
+
for (const { row } of rows) {
|
|
294
|
+
if (!byPath.has(row.relativePath))
|
|
295
|
+
byPath.set(row.relativePath, row);
|
|
296
|
+
}
|
|
297
|
+
const statuses = new Map();
|
|
298
|
+
await Promise.all([...byPath.entries()].map(async ([relativePath, row]) => {
|
|
299
|
+
if (typeof row.bytes !== "number" || typeof row.mtimeMs !== "number") {
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
const absolutePath = resolveExistingPath(path.join(projectRoot, ...relativePath.split("/")), isCaseInsensitiveFileSystem(projectRoot));
|
|
303
|
+
const info = absolutePath === null ? null : await stat(absolutePath).catch(() => null);
|
|
304
|
+
if (!info?.isFile()) {
|
|
305
|
+
statuses.set(relativePath, "missing");
|
|
306
|
+
}
|
|
307
|
+
else if (info.size !== row.bytes || info.mtimeMs !== row.mtimeMs) {
|
|
308
|
+
statuses.set(relativePath, "modified");
|
|
309
|
+
}
|
|
310
|
+
}));
|
|
311
|
+
return statuses;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Count, for each query word, the passages whose keyword text contains it, with one bounded
|
|
315
|
+
* full-text query per word. Cached per index generation: words repeat across an agent's searches.
|
|
316
|
+
*/
|
|
317
|
+
async function queryTermFrequencies(table, tokens, snapshot) {
|
|
318
|
+
const stats = await table.indexStats(FULL_TEXT_INDEX_NAME).catch(() => undefined);
|
|
319
|
+
if (!stats || stats.numUnindexedRows > 0) {
|
|
320
|
+
return null;
|
|
321
|
+
}
|
|
322
|
+
const cacheKey = `${snapshot.tableName}\0${snapshot.manifestFingerprint ?? ""}`;
|
|
323
|
+
let cache = termFrequencyCache.get(cacheKey);
|
|
324
|
+
if (!cache) {
|
|
325
|
+
cache = new Map();
|
|
326
|
+
termFrequencyCache.set(cacheKey, cache);
|
|
327
|
+
while (termFrequencyCache.size > MAX_CACHED_TERM_GENERATIONS) {
|
|
328
|
+
const oldest = termFrequencyCache.keys().next().value;
|
|
329
|
+
if (oldest === undefined)
|
|
330
|
+
break;
|
|
331
|
+
termFrequencyCache.delete(oldest);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
const generationCache = cache;
|
|
335
|
+
const frequencyLimit = Math.min(MAX_TERM_FREQUENCY_LIMIT, Math.max(MIN_TERM_FREQUENCY_LIMIT, Math.ceil(stats.numIndexedRows * TERM_FREQUENCY_SHARE)));
|
|
336
|
+
const unique = [...new Set(tokens)];
|
|
337
|
+
const unmeasuredTerms = new Set(unique.filter((token) => UNSEGMENTED_SCRIPT_PATTERN.test(token)));
|
|
338
|
+
const measured = unique.filter((token) => !unmeasuredTerms.has(token));
|
|
339
|
+
if (measured.length === 0) {
|
|
340
|
+
return null;
|
|
341
|
+
}
|
|
342
|
+
await Promise.all(measured
|
|
343
|
+
.filter((token) => !generationCache.has(token))
|
|
344
|
+
.map(async (token) => {
|
|
345
|
+
const rows = await table
|
|
346
|
+
.search(new MatchQuery(token, "searchText"), "fts", "searchText")
|
|
347
|
+
.select(["id"])
|
|
348
|
+
.limit(frequencyLimit)
|
|
349
|
+
.toArray();
|
|
350
|
+
generationCache.set(token, rows.length);
|
|
351
|
+
}));
|
|
352
|
+
return {
|
|
353
|
+
documentCount: stats.numIndexedRows,
|
|
354
|
+
documentFrequencies: new Map(measured.map((token) => [token, generationCache.get(token) ?? 0])),
|
|
355
|
+
...(unmeasuredTerms.size > 0 ? { unmeasuredTerms } : {}),
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
/** Mark absent words that a candidate still matches fuzzily, so typos are not treated as noise. */
|
|
359
|
+
function withFuzzyMatches(statistics, tokens, candidateRows) {
|
|
360
|
+
if (!statistics) {
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
const measured = [...new Set(tokens)].filter((token) => !statistics.unmeasuredTerms?.has(token));
|
|
364
|
+
// Fuzzy matches only decide whether no query word is known at all; skip the scan otherwise.
|
|
365
|
+
if (measured.some((token) => (statistics.documentFrequencies.get(token) ?? 0) > 0)) {
|
|
366
|
+
return statistics;
|
|
367
|
+
}
|
|
368
|
+
const absent = measured.filter((token) => token.length >= 7);
|
|
369
|
+
if (absent.length === 0 || candidateRows.length === 0) {
|
|
370
|
+
return statistics;
|
|
371
|
+
}
|
|
372
|
+
const prefilters = absent.map((token) => ({ token, possible: fuzzyMatchPrefilter(token) }));
|
|
373
|
+
const fuzzyMatchedTerms = new Set();
|
|
374
|
+
for (const row of candidateRows) {
|
|
375
|
+
const folded = normalizeForMatch(row.searchText);
|
|
376
|
+
const pending = prefilters
|
|
377
|
+
.filter(({ token, possible }) => !fuzzyMatchedTerms.has(token) && possible(folded))
|
|
378
|
+
.map(({ token }) => token);
|
|
379
|
+
if (pending.length === 0) {
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
382
|
+
for (const textToken of tokenize(row.searchText)) {
|
|
383
|
+
for (const token of pending) {
|
|
384
|
+
if (!fuzzyMatchedTerms.has(token) && tokensAreFuzzyRelated(token, textToken)) {
|
|
385
|
+
fuzzyMatchedTerms.add(token);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (fuzzyMatchedTerms.size === absent.length)
|
|
390
|
+
break;
|
|
391
|
+
}
|
|
392
|
+
return { ...statistics, fuzzyMatchedTerms };
|
|
393
|
+
}
|
|
212
394
|
export function vectorCandidateLimit(candidateDemand, profile = "balanced") {
|
|
213
395
|
return Math.min(MAX_VECTOR_CANDIDATES, candidateDepth(candidateDemand, profile));
|
|
214
396
|
}
|
|
@@ -273,7 +455,7 @@ async function expandCitationWithinGeneration(citation, options, config, snapsho
|
|
|
273
455
|
const rows = (await table
|
|
274
456
|
.query()
|
|
275
457
|
.select(SEARCH_COLUMNS)
|
|
276
|
-
.where(
|
|
458
|
+
.where(`${relativePathEquals(target.relativePath)} AND chunkIndex >= ${minimumChunkIndex} AND chunkIndex <= ${maximumChunkIndex}`)
|
|
277
459
|
.toArray());
|
|
278
460
|
throwIfAborted(signal);
|
|
279
461
|
const targetRow = rows.find((row) => row.chunkIndex === target.chunkIndex);
|
|
@@ -287,7 +469,7 @@ async function expandCitationWithinGeneration(citation, options, config, snapsho
|
|
|
287
469
|
passages: [],
|
|
288
470
|
};
|
|
289
471
|
}
|
|
290
|
-
if (citationForRow(targetRow) !== requestedCitation) {
|
|
472
|
+
if (canonicalUserPath(citationForRow(targetRow)) !== canonicalUserPath(requestedCitation)) {
|
|
291
473
|
throw new Error("Citation coordinates do not match the indexed passage.");
|
|
292
474
|
}
|
|
293
475
|
const evidence = evidenceVersionForRow(targetRow, snapshot, knowledgeBaseIdentity(config.projectRoot)?.id ?? null);
|
|
@@ -300,7 +482,7 @@ async function expandCitationWithinGeneration(citation, options, config, snapsho
|
|
|
300
482
|
const headers = (await table
|
|
301
483
|
.query()
|
|
302
484
|
.select(SEARCH_COLUMNS)
|
|
303
|
-
.where(
|
|
485
|
+
.where(`${relativePathEquals(target.relativePath)} AND chunkIndex = ${targetRow.headerChunkIndex}`)
|
|
304
486
|
.limit(1)
|
|
305
487
|
.toArray());
|
|
306
488
|
throwIfAborted(signal);
|
|
@@ -380,7 +562,7 @@ async function lexicalCandidateRows(table, query, options) {
|
|
|
380
562
|
const exactPathMatches = new Set(sourcePathQuery
|
|
381
563
|
? sourcePathRows
|
|
382
564
|
.filter((row) => sourcePathQuery.exactRelativePath === null ||
|
|
383
|
-
row.relativePath === sourcePathQuery.exactRelativePath)
|
|
565
|
+
row.relativePath.normalize("NFC") === sourcePathQuery.exactRelativePath)
|
|
384
566
|
.map(rowKey)
|
|
385
567
|
: []);
|
|
386
568
|
return {
|
|
@@ -416,7 +598,7 @@ async function lexicalCandidateRows(table, query, options) {
|
|
|
416
598
|
const exactPathMatches = new Set(sourcePathQuery
|
|
417
599
|
? sourcePathRows
|
|
418
600
|
.filter((row) => sourcePathQuery.exactRelativePath === null ||
|
|
419
|
-
row.relativePath === sourcePathQuery.exactRelativePath)
|
|
601
|
+
row.relativePath.normalize("NFC") === sourcePathQuery.exactRelativePath)
|
|
420
602
|
.map(rowKey)
|
|
421
603
|
: []);
|
|
422
604
|
return {
|
|
@@ -498,18 +680,34 @@ async function executeSourcePathQuery(table, sourcePathQuery, limit, retrievalPr
|
|
|
498
680
|
: sourcePathQuery.predicate;
|
|
499
681
|
return (await table.query().select(SEARCH_COLUMNS).where(predicate).limit(limit).toArray()).map((row) => ({
|
|
500
682
|
...row,
|
|
501
|
-
_score: sourcePathQuery.exactRelativePath === row.relativePath
|
|
683
|
+
_score: sourcePathQuery.exactRelativePath === row.relativePath.normalize("NFC")
|
|
502
684
|
? Number.MAX_SAFE_INTEGER
|
|
503
685
|
: Number.MAX_SAFE_INTEGER - 1,
|
|
504
686
|
}));
|
|
505
687
|
}
|
|
506
|
-
function
|
|
507
|
-
|
|
688
|
+
function resolveSearchScope(config, scopeName) {
|
|
689
|
+
if (scopeName === undefined) {
|
|
690
|
+
return null;
|
|
691
|
+
}
|
|
692
|
+
const scope = config.scopes[scopeName];
|
|
693
|
+
if (!scope) {
|
|
694
|
+
const names = Object.keys(config.scopes);
|
|
695
|
+
throw new RagmirError("INVALID_ARGUMENT", names.length === 0
|
|
696
|
+
? `Unknown scope "${scopeName}": this knowledge base defines no scopes.`
|
|
697
|
+
: `Unknown scope "${scopeName}". Configured scopes: ${names.join(", ")}.`);
|
|
698
|
+
}
|
|
699
|
+
return scope;
|
|
700
|
+
}
|
|
701
|
+
/** Each include group narrows the search (AND); paths inside one group are alternatives (OR). */
|
|
702
|
+
function searchPredicate(includeGroups, excludePaths, contextPaths) {
|
|
508
703
|
const excludes = normalizePathPrefixes(excludePaths);
|
|
509
704
|
const contexts = normalizeContextPaths(contextPaths);
|
|
510
705
|
const clauses = [];
|
|
511
|
-
|
|
512
|
-
|
|
706
|
+
for (const includePaths of includeGroups) {
|
|
707
|
+
const includes = normalizePathPrefixes(includePaths);
|
|
708
|
+
if (includes.length > 0) {
|
|
709
|
+
clauses.push(`(${includes.map(pathPrefixPredicate).join(" OR ")})`);
|
|
710
|
+
}
|
|
513
711
|
}
|
|
514
712
|
if (excludes.length > 0) {
|
|
515
713
|
clauses.push(`NOT (${excludes.map(pathPrefixPredicate).join(" OR ")})`);
|
|
@@ -520,15 +718,19 @@ function searchPredicate(includePaths, excludePaths, contextPaths) {
|
|
|
520
718
|
return clauses.length === 0 ? null : clauses.join(" AND ");
|
|
521
719
|
}
|
|
522
720
|
function normalizePathPrefixes(prefixes) {
|
|
523
|
-
return [
|
|
524
|
-
...new Set((prefixes ?? [])
|
|
525
|
-
.map((prefix) => prefix.trim().replaceAll("\\", "/").replace(/^\.\//u, ""))
|
|
526
|
-
.map((prefix) => prefix.replace(/\/+$/u, ""))
|
|
527
|
-
.filter(Boolean)),
|
|
528
|
-
];
|
|
721
|
+
return [...new Set((prefixes ?? []).map(canonicalUserPath).filter(Boolean))];
|
|
529
722
|
}
|
|
723
|
+
/** Match the canonical path and its decomposed form, which indexes built before 6.2 may hold. */
|
|
530
724
|
function pathPrefixPredicate(prefix) {
|
|
531
|
-
return
|
|
725
|
+
return unicodePathVariants(prefix)
|
|
726
|
+
.map((variant) => `(relativePath = ${sqlString(variant)} OR starts_with(relativePath, ${sqlString(`${variant}/`)}))`)
|
|
727
|
+
.join(" OR ");
|
|
728
|
+
}
|
|
729
|
+
function relativePathEquals(relativePath) {
|
|
730
|
+
const variants = unicodePathVariants(relativePath);
|
|
731
|
+
return variants.length === 1
|
|
732
|
+
? `relativePath = ${sqlString(relativePath)}`
|
|
733
|
+
: `relativePath IN (${variants.map(sqlString).join(", ")})`;
|
|
532
734
|
}
|
|
533
735
|
function normalizeContextPaths(contextPaths) {
|
|
534
736
|
return [...new Set((contextPaths ?? []).map((value) => value.trim()).filter(Boolean))];
|
|
@@ -665,7 +867,7 @@ function parseCitationTarget(citation) {
|
|
|
665
867
|
throw new Error("Citation must end with a Ragmir chunk suffix such as `#3`.");
|
|
666
868
|
}
|
|
667
869
|
const chunkIndex = Number.parseInt(chunkIndexText, 10);
|
|
668
|
-
let relativePath = citation.slice(0, match.index);
|
|
870
|
+
let relativePath = canonicalUserPath(citation.slice(0, match.index));
|
|
669
871
|
relativePath = stripCitationCoordinates(relativePath);
|
|
670
872
|
if (!relativePath) {
|
|
671
873
|
throw new Error("Citation must include a source path before its chunk suffix.");
|
|
@@ -718,17 +920,30 @@ function lexicalQuery(query) {
|
|
|
718
920
|
.map((match) => match[0])
|
|
719
921
|
.filter(Boolean);
|
|
720
922
|
const identifiers = [...new Set(identifierTerms)];
|
|
721
|
-
|
|
923
|
+
// Only identifiers with a digit, underscore, or dot must match exactly. Hyphenated words such as
|
|
924
|
+
// "t-score" or the French "est-elle" boost the passages that contain them without excluding others.
|
|
925
|
+
const exactIdentifiers = identifiers.filter((identifier) => /[\d_.]/u.test(identifier));
|
|
926
|
+
const hyphenatedWords = identifiers.filter((identifier) => !/[\d_.]/u.test(identifier));
|
|
927
|
+
const rankedQuery = hyphenatedWords.length === 0
|
|
928
|
+
? broadQuery
|
|
929
|
+
: new BooleanQuery([
|
|
930
|
+
[Occur.Should, broadQuery],
|
|
931
|
+
...hyphenatedWords.map((word) => [
|
|
932
|
+
Occur.Should,
|
|
933
|
+
new PhraseQuery(word, "searchText"),
|
|
934
|
+
]),
|
|
935
|
+
]);
|
|
936
|
+
const firstIdentifier = exactIdentifiers[0];
|
|
722
937
|
if (firstIdentifier !== undefined) {
|
|
723
938
|
const firstExactIdentifierQuery = new PhraseQuery(firstIdentifier, "searchText");
|
|
724
939
|
const exactIdentifierQueries = [
|
|
725
940
|
firstExactIdentifierQuery,
|
|
726
|
-
...
|
|
941
|
+
...exactIdentifiers.slice(1).map((identifier) => new PhraseQuery(identifier, "searchText")),
|
|
727
942
|
];
|
|
728
943
|
const exactIdentifierQuery = exactIdentifierQueries.length === 1
|
|
729
944
|
? firstExactIdentifierQuery
|
|
730
945
|
: new BooleanQuery(exactIdentifierQueries.map((item) => [Occur.Should, item]));
|
|
731
|
-
const fuzzyIdentifierQuery = new BooleanQuery(
|
|
946
|
+
const fuzzyIdentifierQuery = new BooleanQuery(exactIdentifiers.map((identifier) => [
|
|
732
947
|
Occur.Should,
|
|
733
948
|
new MatchQuery(identifier, "searchText", {
|
|
734
949
|
boost: 2,
|
|
@@ -737,11 +952,11 @@ function lexicalQuery(query) {
|
|
|
737
952
|
prefixLength: 3,
|
|
738
953
|
}),
|
|
739
954
|
]));
|
|
740
|
-
supplemental.push(fuzzyIdentifierQuery,
|
|
955
|
+
supplemental.push(fuzzyIdentifierQuery, rankedQuery);
|
|
741
956
|
return {
|
|
742
957
|
primary: new BooleanQuery([
|
|
743
958
|
[Occur.Must, exactIdentifierQuery],
|
|
744
|
-
[Occur.Should,
|
|
959
|
+
[Occur.Should, rankedQuery],
|
|
745
960
|
]),
|
|
746
961
|
supplemental,
|
|
747
962
|
};
|
|
@@ -761,7 +976,7 @@ function lexicalQuery(query) {
|
|
|
761
976
|
}));
|
|
762
977
|
}
|
|
763
978
|
return {
|
|
764
|
-
primary:
|
|
979
|
+
primary: rankedQuery,
|
|
765
980
|
supplemental,
|
|
766
981
|
};
|
|
767
982
|
}
|
|
@@ -769,7 +984,7 @@ function isFuzzyLexicalTerm(term) {
|
|
|
769
984
|
return term.length >= 7 && /^[a-z0-9_.-]+$/u.test(term);
|
|
770
985
|
}
|
|
771
986
|
function sourcePathPredicateForQuery(query) {
|
|
772
|
-
const normalized = query
|
|
987
|
+
const normalized = canonicalUserPath(query);
|
|
773
988
|
if (!SOURCE_PATH_QUERY_PATTERN.test(normalized)) {
|
|
774
989
|
return null;
|
|
775
990
|
}
|
|
@@ -777,11 +992,10 @@ function sourcePathPredicateForQuery(query) {
|
|
|
777
992
|
if (!fileName) {
|
|
778
993
|
return null;
|
|
779
994
|
}
|
|
780
|
-
const clauses = new Set([
|
|
781
|
-
|
|
782
|
-
`relativePath
|
|
783
|
-
|
|
784
|
-
]);
|
|
995
|
+
const clauses = new Set([normalized, fileName].flatMap((value) => [
|
|
996
|
+
relativePathEquals(value),
|
|
997
|
+
...unicodePathVariants(fileName).map((variant) => `ends_with(relativePath, ${sqlString(`/${variant}`)})`),
|
|
998
|
+
]));
|
|
785
999
|
return {
|
|
786
1000
|
predicate: `(${[...clauses].join(" OR ")})`,
|
|
787
1001
|
exactRelativePath: normalized.includes("/") ? normalized : null,
|