@ssmeo/mcp-local-rag 0.18.10 → 0.18.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.de.md +23 -2
- package/README.es.md +23 -2
- package/README.fr.md +23 -2
- package/README.md +34 -2
- package/README.pt-BR.md +23 -2
- package/README.zh-CN.md +23 -2
- package/dist/cli/common.d.ts.map +1 -1
- package/dist/cli/common.js +15 -2
- package/dist/cli/common.js.map +1 -1
- package/dist/cli/options.d.ts +19 -0
- package/dist/cli/options.d.ts.map +1 -1
- package/dist/cli/options.js +72 -8
- package/dist/cli/options.js.map +1 -1
- package/dist/embedder/remote.d.ts.map +1 -1
- package/dist/embedder/remote.js +12 -5
- package/dist/embedder/remote.js.map +1 -1
- package/dist/rerank/command.d.ts +21 -0
- package/dist/rerank/command.d.ts.map +1 -0
- package/dist/rerank/command.js +20 -0
- package/dist/rerank/command.js.map +1 -0
- package/dist/rerank/index.d.ts +21 -0
- package/dist/rerank/index.d.ts.map +1 -0
- package/dist/rerank/index.js +104 -0
- package/dist/rerank/index.js.map +1 -0
- package/dist/rerank/response.d.ts +29 -0
- package/dist/rerank/response.d.ts.map +1 -0
- package/dist/rerank/response.js +77 -0
- package/dist/rerank/response.js.map +1 -0
- package/dist/server/index.d.ts +5 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +59 -27
- package/dist/server/index.js.map +1 -1
- package/dist/server/tool-definitions.js +1 -1
- package/dist/server/tool-definitions.js.map +1 -1
- package/dist/server/tool-input.d.ts.map +1 -1
- package/dist/server/tool-input.js +5 -5
- package/dist/server/tool-input.js.map +1 -1
- package/dist/server/types.d.ts +4 -0
- package/dist/server/types.d.ts.map +1 -1
- package/dist/server-main.d.ts +13 -22
- package/dist/server-main.d.ts.map +1 -1
- package/dist/server-main.js +50 -59
- package/dist/server-main.js.map +1 -1
- package/dist/utils/limits.d.ts +16 -1
- package/dist/utils/limits.d.ts.map +1 -1
- package/dist/utils/limits.js +16 -1
- package/dist/utils/limits.js.map +1 -1
- package/dist/vectordb/types.d.ts +4 -1
- package/dist/vectordb/types.d.ts.map +1 -1
- package/dist/vectordb/types.js.map +1 -1
- package/package.json +1 -1
- package/skills/mcp-local-rag/SKILL.md +3 -3
- package/skills/mcp-local-rag/references/cli-reference.md +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"remote.js","sourceRoot":"","sources":["../../src/embedder/remote.ts"],"names":[],"mappings":"AAAA,oDAAoD;AACpD,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAE3E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAE7C,yCAAyC;AACzC,MAAM,OAAO,oBAAqB,SAAQ,QAAQ;IAChD,YAAY,OAAe,EAAE,OAA2B;QACtD,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;QAC/C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAA;IACpC,CAAC;CACF;AAED,6DAA6D;AAC7D,MAAM,OAAO,6BAA8B,SAAQ,oBAAoB;IAC5D,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,IAAY,EAAE,SAAiB;QACzD,KAAK,CACH,QAAQ,MAAM,0BAA0B,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5F,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAA;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;CACF;AAWD,+CAA+C;AAC/C,4CAA4C;AAC5C
|
|
1
|
+
{"version":3,"file":"remote.js","sourceRoot":"","sources":["../../src/embedder/remote.ts"],"names":[],"mappings":"AAAA,oDAAoD;AACpD,EAAE;AACF,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,uCAAuC;AACvC,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAE3E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAE7C,yCAAyC;AACzC,MAAM,OAAO,oBAAqB,SAAQ,QAAQ;IAChD,YAAY,OAAe,EAAE,OAA2B;QACtD,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;QAC/C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAA;IACpC,CAAC;CACF;AAED,6DAA6D;AAC7D,MAAM,OAAO,6BAA8B,SAAQ,oBAAoB;IAC5D,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,IAAY,EAAE,SAAiB;QACzD,KAAK,CACH,QAAQ,MAAM,0BAA0B,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5F,CAAA;QACD,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAA;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;CACF;AAWD,+CAA+C;AAC/C,4CAA4C;AAC5C,0DAA0D;AAC1D,mDAAmD;AACnD,MAAM,WAAW,GAAG,CAAC,CAAA;AACrB,MAAM,kBAAkB,GAAG,IAAI,CAAA;AAC/B,MAAM,cAAc,GAAG,KAAK,CAAA;AAE5B,SAAS,gBAAgB,CAAC,CAAU;IAClC,wEAAwE;IACxE,2BAA2B;IAC3B,IAAI,CAAC,YAAY,6BAA6B,EAAE,CAAC;QAC/C,OAAO,IAAI,CAAA;IACb,CAAC;IACD,OAAO,CAAC,CAAC,CAAC,YAAY,oBAAoB,CAAC,CAAA;AAC7C,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAc;IACvC,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAA;AACxC,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;IAC1D,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;IACvB,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,4EAA4E;AAC5E,SAAS,SAAS,CAAC,OAAe;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,GAAG,CAAC,IAAI,OAAO,CAAC,CAAA;IACvE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAA;AACxC,CAAC;AAED,KAAK,UAAU,SAAS,CAAI,EAAoB,EAAE,OAAe;IAC/D,IAAI,SAAkB,CAAA;IACtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAA;QACnB,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,SAAS,GAAG,CAAC,CAAA;YACb,IAAI,OAAO,KAAK,WAAW,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,MAAM,CAAC,CAAA;YACT,CAAC;YACD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAA;YAC/B,OAAO,CAAC,KAAK,CACX,mBAAmB,OAAO,oBAAoB,OAAO,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,kBAAkB,IAAI,IAAI,CACvG,CAAA;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,CAAA;QACnB,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAwC,EACxC,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAA;IAC7D,IAAI,CAAC;QACH,OAAO,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IACrC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAA;IACrB,CAAC;AACH,CAAC;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;;;;GAUG;AACH,MAAM,OAAO,cAAc;IACR,MAAM,CAAsB;IACrC,IAAI,GAAkB,IAAI,CAAA;IAC1B,WAAW,GAAY,KAAK,CAAA;IAEpC,YAAY,MAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAM;QACR,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAA;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,oBAAoB,CAC5B,qDAAqD,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAC7E,CAAA;QACH,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAA;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACvB,OAAO,CAAC,KAAK,CACX,iCAAiC,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,IAAI,CAAC,IAAI,GAAG,CACnF,CAAA;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IAClB,CAAC;IAED,0EAA0E;IAC1E,IAAI,UAAU;QACZ,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QACzB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,oBAAoB,CAC5B,kEAAkE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAC1F,CAAA;QACH,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvB,MAAM,OAAO,GAAe,EAAE,CAAA;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACvD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,cAAc,KAAK,CAAC,MAAM,SAAS,CAAC,CAAA;YAC7F,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;YAChC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAA;QACtB,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,QAAQ,CAAC,OAAmB,EAAE,aAAqB;QACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAChE,MAAM,IAAI,oBAAoB,CAC5B,oCAAoC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG;gBAC1F,eAAe,aAAa,YAAY,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAClE,CAAA;QACH,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACvB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;oBAChD,MAAM,IAAI,oBAAoB,CAC5B,gDAAgD,IAAI,CAAC,IAAI,QAAQ;wBAC/D,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,oCAAoC;wBAChF,IAAI,CAAC,MAAM,CAAC,SAAS,CACxB,CAAA;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,MAAgB;QACtC,OAAO,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAClC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,QAAQ,EAAE;gBACxD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,oEAAoE;oBACpE,qEAAqE;oBACrE,sEAAsE;oBACtE,UAAU,EAAE,OAAO;iBACpB;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;gBAChC,MAAM;aACP,CAAC,CAAA;YACF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;gBAC7C,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;oBAClC,MAAM,IAAI,6BAA6B,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBAClF,CAAC;gBACD,MAAM,IAAI,oBAAoB,CAC5B,QAAQ,GAAG,CAAC,MAAM,0BAA0B,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5G,CAAA;YACH,CAAC;YACD,MAAM,OAAO,GAAe,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;YAC5C,OAAO,OAAO,CAAA;QAChB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,CAAA;IACrC,CAAC;CACF"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** An executable and the arguments configured alongside it. */
|
|
2
|
+
export interface RerankCommand {
|
|
3
|
+
/** Program to spawn. Must name a directly executable file, not a shim. */
|
|
4
|
+
executable: string;
|
|
5
|
+
/** Arguments configured by the operator, before the server's own flags. */
|
|
6
|
+
args: string[];
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Splits the configured command on whitespace. Quoting and other shell syntax
|
|
10
|
+
* are deliberately not interpreted: the command is spawned without a shell, so
|
|
11
|
+
* a quote this parser stripped would not mean what the operator expects.
|
|
12
|
+
*
|
|
13
|
+
* Returns undefined when the string carries no token.
|
|
14
|
+
*/
|
|
15
|
+
export declare function parseRerankCommand(command: string): RerankCommand | undefined;
|
|
16
|
+
/**
|
|
17
|
+
* Appends the server's flags after the configured arguments. Each value is its
|
|
18
|
+
* own element, so query text is never read as command syntax.
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildRerankArgv(command: RerankCommand, query: string, top: number): string[];
|
|
21
|
+
//# sourceMappingURL=command.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"command.d.ts","sourceRoot":"","sources":["../../src/rerank/command.ts"],"names":[],"mappings":"AAEA,+DAA+D;AAC/D,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAA;IAClB,2EAA2E;IAC3E,IAAI,EAAE,MAAM,EAAE,CAAA;CACf;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAG7E;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAE5F"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// Parsing of the configured rerank command into an argv vector.
|
|
2
|
+
/**
|
|
3
|
+
* Splits the configured command on whitespace. Quoting and other shell syntax
|
|
4
|
+
* are deliberately not interpreted: the command is spawned without a shell, so
|
|
5
|
+
* a quote this parser stripped would not mean what the operator expects.
|
|
6
|
+
*
|
|
7
|
+
* Returns undefined when the string carries no token.
|
|
8
|
+
*/
|
|
9
|
+
export function parseRerankCommand(command) {
|
|
10
|
+
const [executable, ...args] = command.split(/\s+/).filter((token) => token.length > 0);
|
|
11
|
+
return executable === undefined ? undefined : { executable, args };
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Appends the server's flags after the configured arguments. Each value is its
|
|
15
|
+
* own element, so query text is never read as command syntax.
|
|
16
|
+
*/
|
|
17
|
+
export function buildRerankArgv(command, query, top) {
|
|
18
|
+
return [...command.args, '--query', query, '--top', String(top)];
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"command.js","sourceRoot":"","sources":["../../src/rerank/command.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAUhE;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,MAAM,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACtF,OAAO,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AACpE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAsB,EAAE,KAAa,EAAE,GAAW;IAChF,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;AAClE,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { type RerankResult } from './response.js';
|
|
2
|
+
export type { RerankResult } from './response.js';
|
|
3
|
+
/** One rerank call: the results to hand over, and the command that handles them. */
|
|
4
|
+
export interface RerankRequest {
|
|
5
|
+
/** Search results in their pre-rerank order. Returned as-is on failure. */
|
|
6
|
+
candidates: RerankResult[];
|
|
7
|
+
/** Query text, passed to the command as its own argv element. */
|
|
8
|
+
query: string;
|
|
9
|
+
/** Result count the caller wants, passed through as `--top`. */
|
|
10
|
+
top: number;
|
|
11
|
+
/** Configured command, parsed as a whitespace-separated argv vector. */
|
|
12
|
+
command: string;
|
|
13
|
+
/** Per-call budget. The spawned process is killed when it elapses. */
|
|
14
|
+
timeoutMs: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* What the command produced, or the results unchanged when it could not run or
|
|
18
|
+
* answered with something that is not a valid result set. Never throws.
|
|
19
|
+
*/
|
|
20
|
+
export declare function rerankCandidates(request: RerankRequest): Promise<RerankResult[]>;
|
|
21
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerank/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,YAAY,EAA0B,MAAM,eAAe,CAAA;AAEzE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAEjD,oFAAoF;AACpF,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,EAAE,MAAM,CAAA;IACX,wEAAwE;IACxE,OAAO,EAAE,MAAM,CAAA;IACf,sEAAsE;IACtE,SAAS,EAAE,MAAM,CAAA;CAClB;AA2FD;;;GAGG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAmBtF"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// External reranking: hand the search candidates to a configured command and
|
|
2
|
+
// take its response as an ordering over them.
|
|
3
|
+
//
|
|
4
|
+
// Every failure mode degrades to the ordering the caller already had, the way
|
|
5
|
+
// the FTS fallback in `VectorStore.searchWithKeywordBoost` does. A reranker is
|
|
6
|
+
// an improvement to ranking, so a broken one must not fail the request; a
|
|
7
|
+
// process that fails to start arrives as an `error` event, and leaving that
|
|
8
|
+
// unhandled would take the server process down.
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
10
|
+
import { errorCode } from '../utils/type-guards.js';
|
|
11
|
+
import { buildRerankArgv, parseRerankCommand } from './command.js';
|
|
12
|
+
import { validateRerankResponse } from './response.js';
|
|
13
|
+
/** Complete results in `docs/schema/query-output.schema.json` form. */
|
|
14
|
+
function buildRerankPayload(candidates) {
|
|
15
|
+
return JSON.stringify(candidates);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A missing command, one without execute permission, and a `.cmd` / `.bat` shim
|
|
19
|
+
* on Windows all land here. The operator would otherwise just see reranking not
|
|
20
|
+
* happening, so the line says what the command has to be. Only the error's code
|
|
21
|
+
* is reported: its message repeats the executable.
|
|
22
|
+
*/
|
|
23
|
+
function spawnFailureReason(error) {
|
|
24
|
+
return `the command could not be started (${errorCode(error) ?? 'unknown error'}); it must name a directly executable file, not a .cmd or .bat shim`;
|
|
25
|
+
}
|
|
26
|
+
function runRerankCommand(executable, argv, payload, timeoutMs) {
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
let settled = false;
|
|
29
|
+
const stdoutChunks = [];
|
|
30
|
+
let timer;
|
|
31
|
+
const finish = (result) => {
|
|
32
|
+
if (settled) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
settled = true;
|
|
36
|
+
clearTimeout(timer);
|
|
37
|
+
resolve(result);
|
|
38
|
+
};
|
|
39
|
+
let child;
|
|
40
|
+
try {
|
|
41
|
+
// No shell on any platform: the query travels as its own argv element and
|
|
42
|
+
// nothing can splice it back into a command string. `stderr` is ignored
|
|
43
|
+
// because the child is untrusted and could otherwise write document text
|
|
44
|
+
// into this server's log stream.
|
|
45
|
+
child = spawn(executable, argv, { stdio: ['pipe', 'pipe', 'ignore'] });
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
finish({ ok: false, reason: spawnFailureReason(error) });
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
// The budget itself ends the call. A child that traps the signal, or one
|
|
52
|
+
// whose descendant holds the inherited stdout open, never emits `close`, so
|
|
53
|
+
// waiting for the kill to land would leave the request pending for as long
|
|
54
|
+
// as the child felt like running. Its stdout is dropped and the handle is
|
|
55
|
+
// unreferenced so nothing it does afterwards reaches or holds this server.
|
|
56
|
+
timer = setTimeout(() => {
|
|
57
|
+
child.kill();
|
|
58
|
+
child.stdout?.destroy();
|
|
59
|
+
child.unref();
|
|
60
|
+
finish({ ok: false, reason: `the command timed out after ${timeoutMs}ms and was killed` });
|
|
61
|
+
}, timeoutMs);
|
|
62
|
+
child.stdout?.setEncoding('utf8');
|
|
63
|
+
child.stdout?.on('data', (chunk) => {
|
|
64
|
+
stdoutChunks.push(chunk);
|
|
65
|
+
});
|
|
66
|
+
// A child that exits before reading its input makes this write fail; the
|
|
67
|
+
// exit itself is what gets reported.
|
|
68
|
+
child.stdin?.on('error', () => { });
|
|
69
|
+
child.stdin?.end(payload);
|
|
70
|
+
child.on('error', (error) => {
|
|
71
|
+
finish({ ok: false, reason: spawnFailureReason(error) });
|
|
72
|
+
});
|
|
73
|
+
child.on('close', (code, signal) => {
|
|
74
|
+
if (code === 0) {
|
|
75
|
+
finish({ ok: true, stdout: stdoutChunks.join('') });
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const outcome = code === null ? `was terminated by ${signal}` : `exited with code ${code}`;
|
|
79
|
+
finish({ ok: false, reason: `the command ${outcome}` });
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
function fallback(candidates, reason) {
|
|
84
|
+
console.error(`Rerank: ${reason}; returning the pre-rerank ordering`);
|
|
85
|
+
return candidates;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* What the command produced, or the results unchanged when it could not run or
|
|
89
|
+
* answered with something that is not a valid result set. Never throws.
|
|
90
|
+
*/
|
|
91
|
+
export async function rerankCandidates(request) {
|
|
92
|
+
const { candidates, query, top, command, timeoutMs } = request;
|
|
93
|
+
const parsed = parseRerankCommand(command);
|
|
94
|
+
if (parsed === undefined) {
|
|
95
|
+
return fallback(candidates, 'the configured command is empty');
|
|
96
|
+
}
|
|
97
|
+
const run = await runRerankCommand(parsed.executable, buildRerankArgv(parsed, query, top), buildRerankPayload(candidates), timeoutMs);
|
|
98
|
+
if (!run.ok) {
|
|
99
|
+
return fallback(candidates, run.reason);
|
|
100
|
+
}
|
|
101
|
+
const validated = validateRerankResponse(run.stdout);
|
|
102
|
+
return validated.ok ? validated.results : fallback(candidates, validated.reason);
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerank/index.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,8CAA8C;AAC9C,EAAE;AACF,8EAA8E;AAC9E,+EAA+E;AAC/E,0EAA0E;AAC1E,4EAA4E;AAC5E,gDAAgD;AAEhD,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAClE,OAAO,EAAqB,sBAAsB,EAAE,MAAM,eAAe,CAAA;AAoBzE,uEAAuE;AACvE,SAAS,kBAAkB,CAAC,UAA0B;IACpD,OAAO,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAA;AACnC,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,qCAAqC,SAAS,CAAC,KAAK,CAAC,IAAI,eAAe,qEAAqE,CAAA;AACtJ,CAAC;AAED,SAAS,gBAAgB,CACvB,UAAkB,EAClB,IAAc,EACd,OAAe,EACf,SAAiB;IAEjB,OAAO,IAAI,OAAO,CAAW,CAAC,OAAO,EAAE,EAAE;QACvC,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,IAAI,KAAiC,CAAA;QAErC,MAAM,MAAM,GAAG,CAAC,MAAgB,EAAQ,EAAE;YACxC,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAM;YACR,CAAC;YACD,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,OAAO,CAAC,MAAM,CAAC,CAAA;QACjB,CAAC,CAAA;QAED,IAAI,KAA+B,CAAA;QACnC,IAAI,CAAC;YACH,0EAA0E;YAC1E,wEAAwE;YACxE,yEAAyE;YACzE,iCAAiC;YACjC,KAAK,GAAG,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;QACxE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACxD,OAAM;QACR,CAAC;QAED,yEAAyE;QACzE,4EAA4E;QAC5E,2EAA2E;QAC3E,0EAA0E;QAC1E,2EAA2E;QAC3E,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,KAAK,CAAC,IAAI,EAAE,CAAA;YACZ,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAA;YACvB,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,+BAA+B,SAAS,mBAAmB,EAAE,CAAC,CAAA;QAC5F,CAAC,EAAE,SAAS,CAAC,CAAA;QAEb,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;QACjC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAQ,EAAE;YAC/C,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,yEAAyE;QACzE,qCAAqC;QACrC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,EAAE,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;QACxC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;QAEzB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAQ,EAAE;YAChC,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1D,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAQ,EAAE;YACvC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;gBACnD,OAAM;YACR,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC,CAAC,oBAAoB,IAAI,EAAE,CAAA;YAC1F,MAAM,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,OAAO,EAAE,EAAE,CAAC,CAAA;QACzD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,QAAQ,CAAI,UAAe,EAAE,MAAc;IAClD,OAAO,CAAC,KAAK,CAAC,WAAW,MAAM,qCAAqC,CAAC,CAAA;IACrE,OAAO,UAAU,CAAA;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAsB;IAC3D,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAA;IAC9D,MAAM,MAAM,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAA;IAC1C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAChC,MAAM,CAAC,UAAU,EACjB,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,EACnC,kBAAkB,CAAC,UAAU,CAAC,EAC9B,SAAS,CACV,CAAA;IACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACzC,CAAC;IAED,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACpD,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAA;AAClF,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** One result, as `docs/schema/query-output.schema.json` defines it. */
|
|
2
|
+
export interface RerankResult {
|
|
3
|
+
filePath: string;
|
|
4
|
+
chunkIndex: number;
|
|
5
|
+
text: string;
|
|
6
|
+
score: number;
|
|
7
|
+
fileTitle: string | null;
|
|
8
|
+
images: {
|
|
9
|
+
imageIndex: number;
|
|
10
|
+
mimeType: 'image/png' | 'image/jpeg';
|
|
11
|
+
data: string;
|
|
12
|
+
}[];
|
|
13
|
+
source?: string;
|
|
14
|
+
[key: string]: unknown;
|
|
15
|
+
}
|
|
16
|
+
/** The reason reaches stderr, so it names no document text and no path. */
|
|
17
|
+
export type RerankMatch = {
|
|
18
|
+
ok: true;
|
|
19
|
+
results: RerankResult[];
|
|
20
|
+
} | {
|
|
21
|
+
ok: false;
|
|
22
|
+
reason: string;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* An empty array, a shorter array, a chunk the server did not send and a
|
|
26
|
+
* rewritten text are all valid answers; only the form is checked.
|
|
27
|
+
*/
|
|
28
|
+
export declare function validateRerankResponse(stdout: string): RerankMatch;
|
|
29
|
+
//# sourceMappingURL=response.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../src/rerank/response.ts"],"names":[],"mappings":"AAKA,wEAAwE;AACxE,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,WAAW,GAAG,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IACpF,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB;AAED,2EAA2E;AAC3E,MAAM,MAAM,WAAW,GAAG;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,YAAY,EAAE,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AA4D/F;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAiBlE"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// The server sends `docs/schema/query-output.schema.json` and accepts the same
|
|
2
|
+
// shape back. Count, identity and repetition are the command's decisions.
|
|
3
|
+
import { isInteger, isRecord } from '../utils/type-guards.js';
|
|
4
|
+
const MIME_TYPES = new Set(['image/png', 'image/jpeg']);
|
|
5
|
+
function isNonNegativeInteger(value) {
|
|
6
|
+
return isInteger(value) && typeof value === 'number' && value >= 0;
|
|
7
|
+
}
|
|
8
|
+
function parseArray(stdout) {
|
|
9
|
+
let parsed;
|
|
10
|
+
try {
|
|
11
|
+
parsed = JSON.parse(stdout);
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
return Array.isArray(parsed) ? parsed : undefined;
|
|
17
|
+
}
|
|
18
|
+
function imagesValid(value) {
|
|
19
|
+
if (!Array.isArray(value)) {
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
return value.every((image) => isRecord(image) &&
|
|
23
|
+
isNonNegativeInteger(image['imageIndex']) &&
|
|
24
|
+
typeof image['mimeType'] === 'string' &&
|
|
25
|
+
MIME_TYPES.has(image['mimeType']) &&
|
|
26
|
+
typeof image['data'] === 'string');
|
|
27
|
+
}
|
|
28
|
+
function isRerankResult(item) {
|
|
29
|
+
if (!isRecord(item)) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
if (typeof item['filePath'] !== 'string') {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
if (!isNonNegativeInteger(item['chunkIndex'])) {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
if (typeof item['text'] !== 'string') {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
if (typeof item['score'] !== 'number' || !Number.isFinite(item['score'])) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
if (item['fileTitle'] !== null && typeof item['fileTitle'] !== 'string') {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
if (!imagesValid(item['images'])) {
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
if (item['source'] !== undefined && typeof item['source'] !== 'string') {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* An empty array, a shorter array, a chunk the server did not send and a
|
|
57
|
+
* rewritten text are all valid answers; only the form is checked.
|
|
58
|
+
*/
|
|
59
|
+
export function validateRerankResponse(stdout) {
|
|
60
|
+
const items = parseArray(stdout);
|
|
61
|
+
if (items === undefined) {
|
|
62
|
+
return { ok: false, reason: 'output was not a JSON array' };
|
|
63
|
+
}
|
|
64
|
+
for (const [index, item] of items.entries()) {
|
|
65
|
+
if (!isRerankResult(item)) {
|
|
66
|
+
return { ok: false, reason: `output item ${index} did not match the result schema` };
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const results = [];
|
|
70
|
+
for (const item of items) {
|
|
71
|
+
if (isRerankResult(item)) {
|
|
72
|
+
results.push(item);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { ok: true, results };
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=response.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response.js","sourceRoot":"","sources":["../../src/rerank/response.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,0EAA0E;AAE1E,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAA;AAiB7D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAA;AAEvD,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,CAAC,CAAA;AACpE,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;AACnD,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,KAAK,CAAC,KAAK,CAChB,CAAC,KAAK,EAAE,EAAE,CACR,QAAQ,CAAC,KAAK,CAAC;QACf,oBAAoB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACzC,OAAO,KAAK,CAAC,UAAU,CAAC,KAAK,QAAQ;QACrC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,CACpC,CAAA;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAa;IACnC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,QAAQ,EAAE,CAAC;QACzC,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;QACrC,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QACzE,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,QAAQ,EAAE,CAAC;QACxE,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QACjC,OAAO,KAAK,CAAA;IACd,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,QAAQ,EAAE,CAAC;QACvE,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;IAChC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAA;IAC7D,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,KAAK,kCAAkC,EAAE,CAAA;QACtF,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AAC9B,CAAC"}
|
package/dist/server/index.d.ts
CHANGED
|
@@ -40,6 +40,9 @@ export declare class RAGServer {
|
|
|
40
40
|
private readonly maxFileSize;
|
|
41
41
|
private readonly device;
|
|
42
42
|
private readonly storeImages;
|
|
43
|
+
/** Configured reranker command. Undefined leaves query_documents unchanged. */
|
|
44
|
+
private readonly rerankCommand;
|
|
45
|
+
private readonly rerankTimeoutMs;
|
|
43
46
|
/**
|
|
44
47
|
* The one current-or-latest sync job this process retains. A new `sync_start`
|
|
45
48
|
* replaces a terminal record, so the older id becomes unknown; there is no
|
|
@@ -84,6 +87,8 @@ export declare class RAGServer {
|
|
|
84
87
|
/**
|
|
85
88
|
* query_documents tool handler
|
|
86
89
|
*/
|
|
90
|
+
/** Hands the finished results to the configured command and takes its answer. */
|
|
91
|
+
private applyReranker;
|
|
87
92
|
handleQueryDocuments(args: QueryDocumentsInput): Promise<{
|
|
88
93
|
content: QueryContent;
|
|
89
94
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+CAA+C,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+CAA+C,CAAA;AAiE9E,OAAO,EAKL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAGzB,MAAM,kBAAkB,CAAA;AAazB,OAAO,KAAK,EAIV,eAAe,EAGf,cAAc,EAEd,mBAAmB,EACnB,eAAe,EAGf,cAAc,EACd,eAAe,EAEhB,MAAM,YAAY,CAAA;AAwBnB,KAAK,YAAY,GAAG,CAAC,mBAAmB,EAAE,GAAG,eAAe,EAAE,CAAC,CAAA;AAkG/D,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA2B;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;IAC5C;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;IAC/C,oFAAoF;IACpF,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IAEjC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAU;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA4B;IACxD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAQ;IACvC;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoB;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAQ;IACxC;;;;OAIG;IACH,OAAO,CAAC,OAAO,CAAgC;IAC/C;;;;OAIG;IACH,OAAO,CAAC,gBAAgB,CAAiB;IAEzC,YAAY,MAAM,EAAE,eAAe,EA2ElC;IAED;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAMtB,mFAAmF;IACnF,OAAO,CAAC,YAAY;IAIpB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,eAAe;IAIvB;;OAEG;IACH,OAAO,CAAC,aAAa;IAwErB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAWhC;IAED;;OAEG;IACH,iFAAiF;YACnE,aAAa;IAiBrB,oBAAoB,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,CAAA;KAAE,CAAC,CA0GxF;IAED;;;;OAIG;IACG,gBAAgB,CACpB,GAAG,EAAE,OAAO,EACZ,OAAO,GAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO,GACzD,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CAc7C;IAED,4EAA4E;YAC9D,oBAAoB;YAsBpB,uBAAuB;IAwBrC;;;OAGG;YACW,cAAc;YAuBd,UAAU;IA0FxB;;;;OAIG;IACG,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CA4FzF;IAED;;;;;;;;;OASG;IACG,eAAe,CAAC,KAAK,GAAE,cAAmB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CAqD7F;IAED;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CAuBhE;IAED;;;;OAIG;IACG,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CA2DhF;IAED;;;OAGG;IACG,wBAAwB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CA4CxF;IAED,yFAAyF;YAC3E,qBAAqB;IAYnC;;;;;;;OAOG;IAKH,OAAO,CAAC,uBAAuB,CAAsB;IACrD,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,mBAAmB,CAAI;YAEjB,sBAAsB;IAsBpC;;;;;OAKG;IACH,OAAO,CAAC,aAAa;IA8Bf,eAAe,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CA6BxF;IAED;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,mBAAmB,EAAE,CAAA;KAAE,CAAC,CAW1F;IAED,kFAAkF;IAClF,OAAO,CAAC,aAAa;IAQrB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;;;OAKG;YACW,UAAU;IAyFxB;;;;;;;;;;;;OAYG;YACW,iBAAiB;IAoB/B;;;;;;;;OAQG;IACG,OAAO,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjD;IAED;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzB;IAED;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;CACF"}
|
package/dist/server/index.js
CHANGED
|
@@ -18,9 +18,10 @@ import { buildPreparedFileVectorChunks, prepareFileForIngest, } from '../ingest/
|
|
|
18
18
|
import { parseHtml } from '../parser/html-parser.js';
|
|
19
19
|
import { DocumentParser, ValidationError } from '../parser/index.js';
|
|
20
20
|
import { extractMarkdownTitle, extractTxtTitle } from '../parser/title-extractor.js';
|
|
21
|
+
import { rerankCandidates } from '../rerank/index.js';
|
|
21
22
|
import { displayPath } from '../utils/base-dirs.js';
|
|
22
23
|
import { toError } from '../utils/errors.js';
|
|
23
|
-
import { MAX_SCAN_DEPTH } from '../utils/limits.js';
|
|
24
|
+
import { DEFAULT_RERANK_TIMEOUT_MS, MAX_QUERY_LIMIT, MAX_SCAN_DEPTH, RERANK_CANDIDATE_MULTIPLIER, } from '../utils/limits.js';
|
|
24
25
|
import { checkRawDataArtifacts, extractSourceFromPath, generateMetaJsonPath, generateRawDataPath, isEnoent, isManagedRawDataPath, isPathInRawDataDir, isPathInRawDataDirLexical, loadMetaJson, saveMetaJson, saveRawData, } from '../utils/raw-data-utils.js';
|
|
25
26
|
import { bfsCollectSupportedFiles, canonicalizeRequestedPath, classifyRequestedPath, } from '../utils/scan.js';
|
|
26
27
|
import { nonAbsolutePrefixes } from '../utils/scope-match.js';
|
|
@@ -154,6 +155,9 @@ export class RAGServer {
|
|
|
154
155
|
maxFileSize;
|
|
155
156
|
device;
|
|
156
157
|
storeImages;
|
|
158
|
+
/** Configured reranker command. Undefined leaves query_documents unchanged. */
|
|
159
|
+
rerankCommand;
|
|
160
|
+
rerankTimeoutMs;
|
|
157
161
|
/**
|
|
158
162
|
* The one current-or-latest sync job this process retains. A new `sync_start`
|
|
159
163
|
* replaces a terminal record, so the older id becomes unknown; there is no
|
|
@@ -185,6 +189,8 @@ export class RAGServer {
|
|
|
185
189
|
this.maxFileSize = config.maxFileSize;
|
|
186
190
|
this.device = config.device;
|
|
187
191
|
this.storeImages = config.storeImages ?? false;
|
|
192
|
+
this.rerankCommand = config.rerankCommand;
|
|
193
|
+
this.rerankTimeoutMs = config.rerankTimeoutMs ?? DEFAULT_RERANK_TIMEOUT_MS;
|
|
188
194
|
this.excludePaths = [`${resolve(this.dbPath)}${sep}`, `${resolve(this.cacheDir)}${sep}`];
|
|
189
195
|
this.server = new Server({ name: 'rag-mcp-server', version: packageVersion }, { capabilities: { tools: {} } });
|
|
190
196
|
// Component initialization
|
|
@@ -356,40 +362,44 @@ export class RAGServer {
|
|
|
356
362
|
/**
|
|
357
363
|
* query_documents tool handler
|
|
358
364
|
*/
|
|
365
|
+
/** Hands the finished results to the configured command and takes its answer. */
|
|
366
|
+
async applyReranker(results, query, limit) {
|
|
367
|
+
if (this.rerankCommand === undefined) {
|
|
368
|
+
return results;
|
|
369
|
+
}
|
|
370
|
+
return rerankCandidates({
|
|
371
|
+
candidates: results,
|
|
372
|
+
query,
|
|
373
|
+
top: limit,
|
|
374
|
+
command: this.rerankCommand,
|
|
375
|
+
timeoutMs: this.rerankTimeoutMs,
|
|
376
|
+
});
|
|
377
|
+
}
|
|
359
378
|
async handleQueryDocuments(args) {
|
|
360
379
|
this.maybeAutoSync();
|
|
361
380
|
// query_documents reads only LanceDB, so it stays callable in degraded
|
|
362
381
|
// mode; `withWarnings` and `status` remain the diagnostic surface.
|
|
363
382
|
const queryVector = await this.embedder.embed(args.query);
|
|
383
|
+
// A reranker only improves on what the search returned, so it is given more
|
|
384
|
+
// candidates than the caller asked for. `search` applies every filter and
|
|
385
|
+
// caps at `limit`, so the expansion has to be requested up front.
|
|
386
|
+
const limit = args.limit ?? 10;
|
|
387
|
+
const candidateLimit = this.rerankCommand === undefined
|
|
388
|
+
? limit
|
|
389
|
+
: Math.min(limit * RERANK_CANDIDATE_MULTIPLIER, MAX_QUERY_LIMIT);
|
|
364
390
|
// `args.scope` is parser-validated; array-wrap without re-validating, and
|
|
365
391
|
// omit the key when absent (exactOptionalPropertyTypes) to keep the scope-absent path.
|
|
366
|
-
const
|
|
392
|
+
const candidates = await this.vectorStore.search(queryVector, {
|
|
367
393
|
queryText: args.query,
|
|
368
|
-
limit:
|
|
394
|
+
limit: candidateLimit,
|
|
369
395
|
...(args.scope !== undefined ? { scope: toArray(args.scope) } : {}),
|
|
370
396
|
});
|
|
371
|
-
//
|
|
372
|
-
const results = searchResults.map((result) => {
|
|
373
|
-
const queryResult = {
|
|
374
|
-
filePath: result.filePath,
|
|
375
|
-
chunkIndex: result.chunkIndex,
|
|
376
|
-
text: result.text,
|
|
377
|
-
score: result.score,
|
|
378
|
-
fileTitle: result.fileTitle ?? null,
|
|
379
|
-
};
|
|
380
|
-
if (isManagedRawDataPath(result.filePath, this.dbPath)) {
|
|
381
|
-
const source = extractSourceFromPath(result.filePath);
|
|
382
|
-
if (source) {
|
|
383
|
-
queryResult.source = source;
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
return queryResult;
|
|
387
|
-
});
|
|
388
|
-
let hydratedRows = [];
|
|
397
|
+
// Hydrated before the command runs, so its answer needs no identity lookup.
|
|
389
398
|
let attachmentWarning = null;
|
|
399
|
+
let attachmentsById = new Map();
|
|
390
400
|
try {
|
|
391
|
-
const hydration = await this.vectorStore.hydrateVisualAttachments(
|
|
392
|
-
|
|
401
|
+
const hydration = await this.vectorStore.hydrateVisualAttachments(candidates);
|
|
402
|
+
attachmentsById = new Map(hydration.rows.map((row) => [row.id, row.attachments]));
|
|
393
403
|
if (hydration.omittedCount > 0) {
|
|
394
404
|
attachmentWarning = attachmentOmissionWarning(hydration.omittedCount);
|
|
395
405
|
}
|
|
@@ -397,15 +407,37 @@ export class RAGServer {
|
|
|
397
407
|
catch {
|
|
398
408
|
attachmentWarning = attachmentHydrationFailureWarning();
|
|
399
409
|
}
|
|
410
|
+
// Format results with source restoration for raw-data files
|
|
411
|
+
const retrieved = candidates.map((candidate) => {
|
|
412
|
+
const result = {
|
|
413
|
+
filePath: candidate.filePath,
|
|
414
|
+
chunkIndex: candidate.chunkIndex,
|
|
415
|
+
text: candidate.text,
|
|
416
|
+
score: candidate.score,
|
|
417
|
+
fileTitle: candidate.fileTitle ?? null,
|
|
418
|
+
images: attachmentsById.get(candidate.id) ?? [],
|
|
419
|
+
};
|
|
420
|
+
if (isManagedRawDataPath(candidate.filePath, this.dbPath)) {
|
|
421
|
+
const source = extractSourceFromPath(candidate.filePath);
|
|
422
|
+
if (source) {
|
|
423
|
+
result.source = source;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return result;
|
|
427
|
+
});
|
|
428
|
+
// `search` already applied the limit, so only the expanded set is trimmed.
|
|
429
|
+
const results = this.rerankCommand === undefined
|
|
430
|
+
? retrieved
|
|
431
|
+
: (await this.applyReranker(retrieved, args.query, limit)).slice(0, limit);
|
|
400
432
|
const content = [
|
|
401
433
|
{
|
|
402
434
|
type: 'text',
|
|
403
|
-
|
|
435
|
+
// Attachments go out as image blocks below, never inside this JSON.
|
|
436
|
+
text: JSON.stringify(results.map(({ images: _images, ...rest }) => rest), null, 2),
|
|
404
437
|
},
|
|
405
438
|
];
|
|
406
|
-
const
|
|
407
|
-
|
|
408
|
-
const attachments = attachmentsByIdentity.get(searchResults[resultIndex]?.id ?? '') ?? [];
|
|
439
|
+
for (const result of results) {
|
|
440
|
+
const attachments = result.images;
|
|
409
441
|
for (const attachment of attachments) {
|
|
410
442
|
content.push({
|
|
411
443
|
type: 'text',
|