@vivantel/virage-cli 0.1.84 โ 0.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ansi.d.ts +14 -0
- package/dist/ansi.d.ts.map +1 -0
- package/dist/ansi.js +25 -0
- package/dist/ansi.js.map +1 -0
- package/dist/bin/virage.js +85 -22
- package/dist/bin/virage.js.map +1 -1
- package/dist/cli/benchmark.d.ts +1 -0
- package/dist/cli/benchmark.d.ts.map +1 -1
- package/dist/cli/benchmark.js +41 -43
- package/dist/cli/benchmark.js.map +1 -1
- package/dist/cli/check.d.ts +1 -0
- package/dist/cli/check.d.ts.map +1 -1
- package/dist/cli/check.js +20 -16
- package/dist/cli/check.js.map +1 -1
- package/dist/cli/chunks-report.d.ts +1 -1
- package/dist/cli/chunks-report.d.ts.map +1 -1
- package/dist/cli/chunks-report.js +16 -11
- package/dist/cli/chunks-report.js.map +1 -1
- package/dist/cli/eval-suite.d.ts.map +1 -1
- package/dist/cli/eval-suite.js +6 -6
- package/dist/cli/eval-suite.js.map +1 -1
- package/dist/cli/evaluate.d.ts +1 -0
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +37 -34
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/experiment.d.ts +3 -1
- package/dist/cli/experiment.d.ts.map +1 -1
- package/dist/cli/experiment.js +41 -39
- package/dist/cli/experiment.js.map +1 -1
- package/dist/cli/init.d.ts +1 -1
- package/dist/cli/init.d.ts.map +1 -1
- package/dist/cli/init.js +25 -24
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/install-hooks.d.ts +1 -0
- package/dist/cli/install-hooks.d.ts.map +1 -1
- package/dist/cli/install-hooks.js +24 -11
- package/dist/cli/install-hooks.js.map +1 -1
- package/dist/cli/pack.d.ts +1 -0
- package/dist/cli/pack.d.ts.map +1 -1
- package/dist/cli/pack.js +9 -16
- package/dist/cli/pack.js.map +1 -1
- package/dist/cli/query-cmd.d.ts +1 -0
- package/dist/cli/query-cmd.d.ts.map +1 -1
- package/dist/cli/query-cmd.js +26 -21
- package/dist/cli/query-cmd.js.map +1 -1
- package/dist/cli/report.d.ts +1 -1
- package/dist/cli/report.d.ts.map +1 -1
- package/dist/cli/report.js +27 -29
- package/dist/cli/report.js.map +1 -1
- package/dist/cli/store-cmd.d.ts +2 -0
- package/dist/cli/store-cmd.d.ts.map +1 -1
- package/dist/cli/store-cmd.js +37 -33
- package/dist/cli/store-cmd.js.map +1 -1
- package/dist/cli/telemetry.d.ts +6 -6
- package/dist/cli/telemetry.d.ts.map +1 -1
- package/dist/cli/telemetry.js +58 -48
- package/dist/cli/telemetry.js.map +1 -1
- package/dist/cli/update.d.ts +1 -1
- package/dist/cli/update.d.ts.map +1 -1
- package/dist/cli/update.js +3 -6
- package/dist/cli/update.js.map +1 -1
- package/dist/cli/validate.d.ts +1 -1
- package/dist/cli/validate.d.ts.map +1 -1
- package/dist/cli/validate.js +9 -14
- package/dist/cli/validate.js.map +1 -1
- package/dist/cli/viz.d.ts +1 -0
- package/dist/cli/viz.d.ts.map +1 -1
- package/dist/cli/viz.js +15 -14
- package/dist/cli/viz.js.map +1 -1
- package/dist/cli-telemetry.d.ts +12 -0
- package/dist/cli-telemetry.d.ts.map +1 -0
- package/dist/cli-telemetry.js +83 -0
- package/dist/cli-telemetry.js.map +1 -0
- package/dist/dashboard-ui/assets/index-CJcTSfAz.js +1434 -0
- package/dist/dashboard-ui/index.html +1 -1
- package/dist/output.d.ts +13 -9
- package/dist/output.d.ts.map +1 -1
- package/dist/output.js +28 -10
- package/dist/output.js.map +1 -1
- package/dist/progress/progress-bar.d.ts +1 -12
- package/dist/progress/progress-bar.d.ts.map +1 -1
- package/dist/progress/progress-bar.js +2 -14
- package/dist/progress/progress-bar.js.map +1 -1
- package/dist/spinner.d.ts +2 -0
- package/dist/spinner.d.ts.map +1 -0
- package/dist/spinner.js +65 -0
- package/dist/spinner.js.map +1 -0
- package/package.json +3 -2
- package/dist/dashboard-ui/assets/index-BKD-_hV9.js +0 -1434
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/cli/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../../src/cli/benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAS5C,MAAM,YAAY,GAAG;IACnB,gCAAgC;IAChC,0DAA0D;IAC1D,4CAA4C;IAC5C,6DAA6D;IAC7D,uEAAuE;CACxE,CAAC;AAEF,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAA8B;IAE9B,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAE9B,GAAG,CAAC,OAAO,CAAC,6BAA6B,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,KAAK;QAAE,GAAG,CAAC,GAAG,CAAC,eAAe,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACtC,GAAG,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,OAAO,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAE/D,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CACtB,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,EACxB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAChD,CAAC;IAEF,MAAM,WAAW,CACf,YAAY,IAAI,CAAC,MAAM,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAC7D,KAAK,IAAI,EAAE;QACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,EACD,CAAC,CACF,CAAC;IAEF,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,WAAW,CACf,WAAW,IAAI,CAAC,OAAO,oBAAoB,EAC3C,KAAK,IAAI,EAAE;QACT,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3B,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,EACD,CAAC,CACF,CAAC;IACF,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEhC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,gBAAgB,GAAG,IAAI,GAAG,GAAG,CAAC;IAEpC,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,OAAO,QAAQ,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;QAC9C,MAAM,WAAW,CACf,wBAAwB,IAAI,CAAC,OAAO,SAAS,EAC7C,KAAK,IAAI,EAAE;YACT,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,CAAC;YAClC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,cAAc,GAAG,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC;QAC/C,CAAC,EACD,CAAC,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GACR,QAAQ,CAAC,UAAU,GAAG,CAAC;QACrB,CAAC,CAAC,QAAQ,CAAC,UAAU;QACrB,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAErD,GAAG,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,CAAC,0BAA0B,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,GAAG,CAAC,IAAI,CAAC,0BAA0B,QAAQ,CAAC,KAAK,IAAI,iBAAiB,EAAE,CAAC,CAAC;IAC1E,GAAG,CAAC,IAAI,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;IAC3C,GAAG,CAAC,OAAO,EAAE,CAAC;IACd,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACnC,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACxD,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACxD,GAAG,CAAC,IAAI,CAAC,0BAA0B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACxD,GAAG,CAAC,IAAI,CAAC,0BAA0B,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IAC7E,IAAI,cAAc,KAAK,IAAI,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QACrD,GAAG,CAAC,OAAO,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,OAAO,SAAS,CAAC,CAAC;QAClD,GAAG,CAAC,IAAI,CACN,0BAA2B,YAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CACnE,CAAC;QACF,GAAG,CAAC,IAAI,CACN,0BAA2B,cAAyB,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CACrE,CAAC;QACF,GAAG,CAAC,IAAI,CACN,0BAA0B,CAAC,IAAI,GAAI,cAAyB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CACtF,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}
|
package/dist/cli/check.d.ts
CHANGED
package/dist/cli/check.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"check.d.ts","sourceRoot":"","sources":["../../src/cli/check.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"check.d.ts","sourceRoot":"","sources":["../../src/cli/check.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAkDhE"}
|
package/dist/cli/check.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import { loadConfig } from "@vivantel/virage-core";
|
|
2
|
+
import { createOut } from "../output.js";
|
|
3
|
+
import { withSpinner } from "../spinner.js";
|
|
2
4
|
export async function runCheck(opts) {
|
|
5
|
+
const out = createOut(opts.verbosity);
|
|
3
6
|
const cfg = await loadConfig(opts.config);
|
|
4
|
-
await
|
|
5
|
-
|
|
7
|
+
const stored = await withSpinner("Reading index metadata", async () => {
|
|
8
|
+
await cfg.vectorStore.initialize();
|
|
9
|
+
return cfg.vectorStore.readMeta?.();
|
|
10
|
+
});
|
|
6
11
|
if (!stored) {
|
|
7
|
-
|
|
12
|
+
out.warn("No embedder metadata found in the index.\n" +
|
|
8
13
|
" This index was built before virage check was available, or the index is empty.\n" +
|
|
9
14
|
" Run `virage index` to write metadata.");
|
|
10
15
|
return;
|
|
@@ -12,29 +17,28 @@ export async function runCheck(opts) {
|
|
|
12
17
|
const builtAt = new Date(stored.createdAt * 1000).toLocaleString();
|
|
13
18
|
const dimMismatch = stored.dimensions !== cfg.embedder.dimensions;
|
|
14
19
|
const modelMismatch = stored.model && cfg.embedder.model && stored.model !== cfg.embedder.model;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
console.log(` Dimensions : ${stored.dimensions}`);
|
|
20
|
+
out.section("๐ Index metadata");
|
|
21
|
+
out.info(` Provider : ${stored.providerName}`);
|
|
22
|
+
out.info(` Model : ${stored.model ?? "(unknown)"}`);
|
|
23
|
+
out.info(` Dimensions : ${stored.dimensions}`);
|
|
20
24
|
if (stored.distanceMetric) {
|
|
21
|
-
|
|
25
|
+
out.info(` Distance : ${stored.distanceMetric}`);
|
|
22
26
|
}
|
|
23
|
-
|
|
24
|
-
|
|
27
|
+
out.info(` Built at : ${builtAt}`);
|
|
28
|
+
out.divider();
|
|
25
29
|
if (dimMismatch || modelMismatch) {
|
|
26
|
-
|
|
30
|
+
out.error("Mismatch with current config:");
|
|
27
31
|
if (dimMismatch) {
|
|
28
|
-
|
|
32
|
+
out.info(` Index dims : ${stored.dimensions} โ config dims: ${cfg.embedder.dimensions}`);
|
|
29
33
|
}
|
|
30
34
|
if (modelMismatch) {
|
|
31
|
-
|
|
35
|
+
out.info(` Index model : ${stored.model} โ config model: ${cfg.embedder.model}`);
|
|
32
36
|
}
|
|
33
|
-
|
|
37
|
+
out.info(" Fix: run `virage index --force` to rebuild the index.");
|
|
34
38
|
process.exitCode = 1;
|
|
35
39
|
}
|
|
36
40
|
else {
|
|
37
|
-
|
|
41
|
+
out.success("Index is compatible with the current embedder config.");
|
|
38
42
|
}
|
|
39
43
|
}
|
|
40
44
|
//# sourceMappingURL=check.js.map
|
package/dist/cli/check.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"check.js","sourceRoot":"","sources":["../../src/cli/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"check.js","sourceRoot":"","sources":["../../src/cli/check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAO5C,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAkB;IAC/C,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACnC,OAAO,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,GAAG,CAAC,IAAI,CACN,4CAA4C;YAC1C,qFAAqF;YACrF,0CAA0C,CAC7C,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;IACnE,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClE,MAAM,aAAa,GACjB,MAAM,CAAC,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;IAE5E,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,KAAK,IAAI,WAAW,EAAE,CAAC,CAAC;IAC1D,GAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,kBAAkB,OAAO,EAAE,CAAC,CAAC;IACtC,GAAG,CAAC,OAAO,EAAE,CAAC;IAEd,IAAI,WAAW,IAAI,aAAa,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC3C,IAAI,WAAW,EAAE,CAAC;YAChB,GAAG,CAAC,IAAI,CACN,oBAAoB,MAAM,CAAC,UAAU,mBAAmB,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,CAClF,CAAC;QACJ,CAAC;QACD,IAAI,aAAa,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CACN,oBAAoB,MAAM,CAAC,KAAK,oBAAoB,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,CACzE,CAAC;QACJ,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;QACrE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare function runChunksReport(dbPath: string): Promise<void>;
|
|
1
|
+
export declare function runChunksReport(dbPath: string, verbosity?: number): Promise<void>;
|
|
2
2
|
//# sourceMappingURL=chunks-report.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunks-report.d.ts","sourceRoot":"","sources":["../../src/cli/chunks-report.ts"],"names":[],"mappings":"AA2BA,wBAAsB,eAAe,
|
|
1
|
+
{"version":3,"file":"chunks-report.d.ts","sourceRoot":"","sources":["../../src/cli/chunks-report.ts"],"names":[],"mappings":"AA2BA,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,SAAS,SAAI,GACZ,OAAO,CAAC,IAAI,CAAC,CA2Cf"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { VirageDb } from "@vivantel/virage-core";
|
|
2
|
-
import {
|
|
2
|
+
import { createOut } from "../output.js";
|
|
3
3
|
function computeCohesion(chunks) {
|
|
4
4
|
const sentenceEnd = /[.!?\n]\s*$/;
|
|
5
5
|
const midSentence = chunks.filter((c) => !sentenceEnd.test(c.content));
|
|
@@ -11,7 +11,8 @@ function computeCohesion(chunks) {
|
|
|
11
11
|
: "Good cohesion";
|
|
12
12
|
return { cohesion, midSentenceCuts: midSentence.length, suggestion };
|
|
13
13
|
}
|
|
14
|
-
export async function runChunksReport(dbPath) {
|
|
14
|
+
export async function runChunksReport(dbPath, verbosity = 0) {
|
|
15
|
+
const out = createOut(verbosity);
|
|
15
16
|
const db = new VirageDb(dbPath);
|
|
16
17
|
const chunks = db.getAllChunks();
|
|
17
18
|
db.close();
|
|
@@ -27,19 +28,23 @@ export async function runChunksReport(dbPath) {
|
|
|
27
28
|
byStrategy.set(strategy, group);
|
|
28
29
|
}
|
|
29
30
|
out.section(`๐ Chunk Cohesion Report (${chunks.length} total chunks)`);
|
|
30
|
-
out.sep("โ", 60);
|
|
31
31
|
for (const [strategy, group] of byStrategy) {
|
|
32
32
|
const sizes = group.map((c) => c.content.length);
|
|
33
33
|
const avgSize = Math.round(sizes.reduce((s, v) => s + v, 0) / sizes.length);
|
|
34
34
|
const { cohesion, midSentenceCuts, suggestion } = computeCohesion(group);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
out.divider();
|
|
36
|
+
out.info(` Strategy: ${strategy}`);
|
|
37
|
+
out.info(` Chunks : ${group.length}`);
|
|
38
|
+
out.info(` Avg size : ${avgSize} chars`);
|
|
39
|
+
out.info(` Cohesion : ${(cohesion * 100).toFixed(1)}%`);
|
|
40
|
+
out.info(` Mid-sentence cuts: ${midSentenceCuts}`);
|
|
41
|
+
if (cohesion >= 0.8) {
|
|
42
|
+
out.success(` ${suggestion}`);
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
out.warn(` ${suggestion}`);
|
|
46
|
+
}
|
|
41
47
|
}
|
|
42
|
-
|
|
43
|
-
out.sep("โ", 60);
|
|
48
|
+
out.divider();
|
|
44
49
|
}
|
|
45
50
|
//# sourceMappingURL=chunks-report.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunks-report.js","sourceRoot":"","sources":["../../src/cli/chunks-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"chunks-report.js","sourceRoot":"","sources":["../../src/cli/chunks-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAOzC,SAAS,eAAe,CAAC,MAAoB;IAK3C,MAAM,WAAW,GAAG,aAAa,CAAC;IAClC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAExD,MAAM,UAAU,GACd,QAAQ,GAAG,GAAG;QACZ,CAAC,CAAC,yFAAyF;QAC3F,CAAC,CAAC,QAAQ,GAAG,GAAG;YACd,CAAC,CAAC,yFAAyF;YAC3F,CAAC,CAAC,eAAe,CAAC;IAExB,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;AACvE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,SAAS,GAAG,CAAC;IAEb,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC;IACjC,EAAE,CAAC,KAAK,EAAE,CAAC;IAEX,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,GAAG,CAAC,KAAK,CACP,uBAAuB,MAAM,+CAA+C,CAC7E,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GACX,KAAK,CAAC,QAAQ,EAAE,QAA+B,IAAI,SAAS,CAAC;QAChE,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,GAAG,CAAC,OAAO,CAAC,6BAA6B,MAAM,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAExE,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5E,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzE,GAAG,CAAC,OAAO,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,yBAAyB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC,yBAAyB,OAAO,QAAQ,CAAC,CAAC;QACnD,GAAG,CAAC,IAAI,CAAC,yBAAyB,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClE,GAAG,CAAC,IAAI,CAAC,0BAA0B,eAAe,EAAE,CAAC,CAAC;QACtD,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YACpB,GAAG,CAAC,OAAO,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,OAAO,UAAU,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../../src/cli/eval-suite.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../../src/cli/eval-suite.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,OAAO,CAAC;IACZ,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuD3E"}
|
package/dist/cli/eval-suite.js
CHANGED
|
@@ -23,11 +23,11 @@ export async function runEvalSuite(opts) {
|
|
|
23
23
|
if (!baselineVariant) {
|
|
24
24
|
throw new Error(`Baseline variant "${suite.baseline}" is not found or is skipped in ${opts.suite}.`);
|
|
25
25
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
const { createOut } = await import("../output.js");
|
|
27
|
+
const out = createOut(opts.verbose);
|
|
28
|
+
out.section("Virage Eval Suite");
|
|
29
|
+
out.dim(` suite: ${opts.suite}`);
|
|
30
|
+
out.dim(` ${activeVariants.length} variant(s) baseline: ${suite.baseline}`);
|
|
31
31
|
const logger = createLogger(opts.verbose);
|
|
32
32
|
const db = new VirageDb(defaultVirageDb());
|
|
33
33
|
let result;
|
|
@@ -42,10 +42,10 @@ export async function runEvalSuite(opts) {
|
|
|
42
42
|
db.close();
|
|
43
43
|
}
|
|
44
44
|
if (opts.json) {
|
|
45
|
+
// Raw JSON output โ intentional console.log for machine-readable stdout
|
|
45
46
|
console.log(JSON.stringify(result, null, 2));
|
|
46
47
|
return;
|
|
47
48
|
}
|
|
48
|
-
console.log("");
|
|
49
49
|
if (!result.ciPassed && opts.ci) {
|
|
50
50
|
process.exit(1);
|
|
51
51
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../../src/cli/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUlD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAyB;IAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAEpC,IAAI,KAAgB,CAAC;IACrB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAc,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,KAAK,EAAE,EAAE;YAC9D,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,oCAAoC,KAAK,CAAC,OAAO,kBAAkB,CACpE,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9E,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,qBAAqB,KAAK,CAAC,QAAQ,mCAAmC,IAAI,CAAC,KAAK,GAAG,CACpF,CAAC;IACJ,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../../src/cli/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUlD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAyB;IAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAEpC,IAAI,KAAgB,CAAC;IACrB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAc,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,KAAK,EAAE,EAAE;YAC9D,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,oCAAoC,KAAK,CAAC,OAAO,kBAAkB,CACpE,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9E,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,qBAAqB,KAAK,CAAC,QAAQ,mCAAmC,IAAI,CAAC,KAAK,GAAG,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACjC,GAAG,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAClC,GAAG,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,MAAM,0BAA0B,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE9E,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC3C,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM;YACN,SAAS,EAAE,IAAI,CAAC,OAAO;SACxB,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,wEAAwE;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QAChC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
|
package/dist/cli/evaluate.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAoBA,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,EAAE,EAAE,OAAO,CAAC;IACZ,KAAK,CAAC,EAAE,WAAW,GAAG,WAAW,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAkBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAqEtE"}
|
package/dist/cli/evaluate.js
CHANGED
|
@@ -1,30 +1,34 @@
|
|
|
1
1
|
import { loadConfig, loadEvalDataset, EvalRunner, EcosystemEvaluator, printEcosystemEvalResult, ExperimentStore, makeRunId, VirageDb, defaultVirageDb, } from "@vivantel/virage-core";
|
|
2
2
|
import { createProgressBar } from "../progress/progress-bar.js";
|
|
3
|
+
import { createOut } from "../output.js";
|
|
4
|
+
import { withSpinner } from "../spinner.js";
|
|
3
5
|
function formatPercent(n) {
|
|
4
6
|
return `${(n * 100).toFixed(1)}%`;
|
|
5
7
|
}
|
|
6
|
-
function printEvalResult(result) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
8
|
+
function printEvalResult(result, verbosity) {
|
|
9
|
+
const out = createOut(verbosity);
|
|
10
|
+
out.section("๐ Retrieval Evaluation Results");
|
|
11
|
+
out.info(` Queries evaluated : ${result.queriesEvaluated}`);
|
|
12
|
+
out.info(` Precision@5 : ${formatPercent(result.precisionAt5)}`);
|
|
13
|
+
out.info(` Precision@10 : ${formatPercent(result.precisionAt10)}`);
|
|
14
|
+
out.info(` Recall@10 : ${formatPercent(result.recallAt10)}`);
|
|
15
|
+
out.info(` MRR : ${result.mrr.toFixed(4)}`);
|
|
16
|
+
out.info(` HitRate@5 : ${formatPercent(result.hitRateAt5)}`);
|
|
17
|
+
out.divider();
|
|
16
18
|
}
|
|
17
19
|
export async function runEvaluate(opts) {
|
|
20
|
+
const verbosity = opts.verbosity ?? 0;
|
|
18
21
|
if (opts.suite === "ecosystem") {
|
|
19
|
-
return runEcosystemEval(opts);
|
|
22
|
+
return runEcosystemEval(opts, verbosity);
|
|
20
23
|
}
|
|
21
|
-
|
|
24
|
+
const out = createOut(verbosity);
|
|
25
|
+
out.info("Loading config...");
|
|
22
26
|
const cfg = await loadConfig(opts.config);
|
|
23
|
-
|
|
27
|
+
out.info(`Loading eval dataset from "${opts.dataset}"...`);
|
|
24
28
|
const dataset = await loadEvalDataset(opts.dataset);
|
|
25
|
-
|
|
26
|
-
await cfg.vectorStore.initialize();
|
|
27
|
-
|
|
29
|
+
out.dim(` Found ${dataset.queries.length} queries`);
|
|
30
|
+
await withSpinner("Initializing vector store", () => cfg.vectorStore.initialize());
|
|
31
|
+
out.info("Running retrieval evaluation...");
|
|
28
32
|
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
29
33
|
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
30
34
|
let evalResult, perQueryRrScores;
|
|
@@ -34,12 +38,11 @@ export async function runEvaluate(opts) {
|
|
|
34
38
|
finally {
|
|
35
39
|
evalBar.stop();
|
|
36
40
|
}
|
|
37
|
-
printEvalResult(evalResult);
|
|
41
|
+
printEvalResult(evalResult, verbosity);
|
|
38
42
|
if (opts.withLlmJudge) {
|
|
39
|
-
|
|
40
|
-
|
|
43
|
+
out.info("LLM-as-judge (RAGAS) requires a judge configured in config.\n" +
|
|
44
|
+
" Add a judge to your pipeline config to enable RAGAS metrics.");
|
|
41
45
|
}
|
|
42
|
-
// Save to experiment store
|
|
43
46
|
const db = new VirageDb(defaultVirageDb());
|
|
44
47
|
try {
|
|
45
48
|
const store = new ExperimentStore(db);
|
|
@@ -52,37 +55,37 @@ export async function runEvaluate(opts) {
|
|
|
52
55
|
perQueryRrScores,
|
|
53
56
|
};
|
|
54
57
|
const runId = await store.save(run);
|
|
55
|
-
|
|
58
|
+
out.success(`Results saved to virage.db (id: ${runId})`);
|
|
56
59
|
}
|
|
57
60
|
finally {
|
|
58
61
|
db.close();
|
|
59
62
|
}
|
|
60
|
-
// CI quality gate
|
|
61
63
|
if (opts.thresholdMrr !== undefined) {
|
|
62
64
|
const passed = evalResult.mrr >= opts.thresholdMrr;
|
|
63
65
|
if (passed) {
|
|
64
|
-
|
|
66
|
+
out.success(`Quality gate passed: MRR ${evalResult.mrr.toFixed(4)} โฅ ${opts.thresholdMrr}`);
|
|
65
67
|
}
|
|
66
68
|
else {
|
|
67
|
-
|
|
69
|
+
out.error(`Quality gate FAILED: MRR ${evalResult.mrr.toFixed(4)} < ${opts.thresholdMrr}`);
|
|
68
70
|
if (opts.ci)
|
|
69
71
|
process.exit(1);
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
}
|
|
73
|
-
async function runEcosystemEval(opts) {
|
|
74
|
-
|
|
75
|
+
async function runEcosystemEval(opts, verbosity) {
|
|
76
|
+
const out = createOut(verbosity);
|
|
77
|
+
out.info("Loading config for ecosystem eval...");
|
|
75
78
|
const cfg = await loadConfig(opts.config);
|
|
76
|
-
|
|
79
|
+
out.info(`Loading ecosystem eval dataset from "${opts.dataset}"...`);
|
|
77
80
|
const rawDataset = JSON.parse(await import("fs/promises").then((fs) => fs.readFile(opts.dataset, "utf-8")));
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
out.dim(` Retrieval queries : ${rawDataset.retrieval.queries.length}`);
|
|
82
|
+
out.dim(` Skill routing : ${rawDataset.skillRouting.length}`);
|
|
80
83
|
if (rawDataset.ragas) {
|
|
81
|
-
|
|
84
|
+
out.dim(` RAGAS queries : ${rawDataset.ragas.queries.length}`);
|
|
82
85
|
}
|
|
83
|
-
await cfg.vectorStore.initialize();
|
|
86
|
+
await withSpinner("Initializing vector store", () => cfg.vectorStore.initialize());
|
|
84
87
|
const evaluator = new EcosystemEvaluator(cfg.vectorStore, cfg.embedder, [], undefined, 10);
|
|
85
|
-
|
|
88
|
+
out.info("Running ecosystem evaluation...");
|
|
86
89
|
const result = await evaluator.run(rawDataset, {
|
|
87
90
|
config: opts.config,
|
|
88
91
|
dataset: opts.dataset,
|
|
@@ -91,10 +94,10 @@ async function runEcosystemEval(opts) {
|
|
|
91
94
|
if (opts.thresholdMrr !== undefined) {
|
|
92
95
|
const passed = result.retrieval.mrr >= opts.thresholdMrr;
|
|
93
96
|
if (passed) {
|
|
94
|
-
|
|
97
|
+
out.success(`MRR gate passed: ${result.retrieval.mrr.toFixed(4)} โฅ ${opts.thresholdMrr}`);
|
|
95
98
|
}
|
|
96
99
|
else {
|
|
97
|
-
|
|
100
|
+
out.error(`MRR gate FAILED: ${result.retrieval.mrr.toFixed(4)} < ${opts.thresholdMrr}`);
|
|
98
101
|
if (opts.ci)
|
|
99
102
|
process.exit(1);
|
|
100
103
|
}
|
package/dist/cli/evaluate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,eAAe,EACf,UAAU,EACV,kBAAkB,EAClB,wBAAwB,EACxB,eAAe,EACf,SAAS,EACT,QAAQ,EACR,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,eAAe,EACf,UAAU,EACV,kBAAkB,EAClB,wBAAwB,EACxB,eAAe,EACf,SAAS,EACT,QAAQ,EACR,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAiB5C,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CAAC,MAAkB,EAAE,SAAiB;IAC5D,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACjC,GAAG,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IAC/C,GAAG,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC7D,GAAG,CAAC,IAAI,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACxE,GAAG,CAAC,IAAI,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACzE,GAAG,CAAC,IAAI,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACtE,GAAG,CAAC,IAAI,CAAC,yBAAyB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3D,GAAG,CAAC,IAAI,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACtE,GAAG,CAAC,OAAO,EAAE,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;IACtC,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QAC/B,OAAO,gBAAgB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAEjC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,GAAG,CAAC,IAAI,CAAC,8BAA8B,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,GAAG,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAEtD,MAAM,WAAW,CAAC,2BAA2B,EAAE,GAAG,EAAE,CAClD,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAC7B,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,IAAI,UAAU,EAAE,gBAAgB,CAAC;IACjC,IAAI,CAAC;QACH,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACnE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC5C,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAEvC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,GAAG,CAAC,IAAI,CACN,+DAA+D;YAC7D,iEAAiE,CACpE,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,GAAG,GAAkB;YACzB,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;YACrB,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YAC1D,UAAU;YACV,gBAAgB;SACjB,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,GAAG,CAAC,OAAO,CAAC,mCAAmC,KAAK,GAAG,CAAC,CAAC;IAC3D,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAED,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC;QACnD,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CACT,4BAA4B,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAC/E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CACP,4BAA4B,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAC/E,CAAC;YACF,IAAI,IAAI,CAAC,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAqB,EACrB,SAAiB;IAEjB,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACjC,GAAG,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,GAAG,CAAC,IAAI,CAAC,wCAAwC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAC3B,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CACtC,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CACnC,CACsB,CAAC;IAC1B,GAAG,CAAC,GAAG,CAAC,0BAA0B,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzE,GAAG,CAAC,GAAG,CAAC,0BAA0B,UAAU,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,GAAG,CAAC,GAAG,CAAC,0BAA0B,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,WAAW,CAAC,2BAA2B,EAAE,GAAG,EAAE,CAClD,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAC7B,CAAC;IAEF,MAAM,SAAS,GAAG,IAAI,kBAAkB,CACtC,GAAG,CAAC,WAAW,EACf,GAAG,CAAC,QAAQ,EACZ,EAAE,EACF,SAAS,EACT,EAAE,CACH,CAAC;IAEF,GAAG,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE;QAC7C,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC,CAAC;IAEH,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAEjC,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC;QACzD,IAAI,MAAM,EAAE,CAAC;YACX,GAAG,CAAC,OAAO,CACT,oBAAoB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAC7E,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,KAAK,CACP,oBAAoB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAC7E,CAAC;YACF,IAAI,IAAI,CAAC,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/cli/experiment.d.ts
CHANGED
|
@@ -2,12 +2,14 @@ export interface ExperimentRunOptions {
|
|
|
2
2
|
name: string;
|
|
3
3
|
config: string;
|
|
4
4
|
dataset: string;
|
|
5
|
+
verbosity?: number;
|
|
5
6
|
}
|
|
6
7
|
export interface ExperimentCompareOptions {
|
|
7
8
|
baseline: string;
|
|
8
9
|
candidate: string;
|
|
10
|
+
verbosity?: number;
|
|
9
11
|
}
|
|
10
12
|
export declare function runExperimentRun(opts: ExperimentRunOptions): Promise<void>;
|
|
11
|
-
export declare function runExperimentList(): Promise<void>;
|
|
13
|
+
export declare function runExperimentList(verbosity?: number): Promise<void>;
|
|
12
14
|
export declare function runExperimentCompare(opts: ExperimentCompareOptions): Promise<void>;
|
|
13
15
|
//# sourceMappingURL=experiment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,IAAI,CAAC,CAgDf;AAED,wBAAsB,iBAAiB,CAAC,SAAS,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAqCpE;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,wBAAwB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAuEf"}
|
package/dist/cli/experiment.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { loadConfig, loadEvalDataset, EvalRunner, ExperimentStore, makeRunId, bootstrapPairedTest, VirageDb, defaultVirageDb, } from "@vivantel/virage-core";
|
|
2
2
|
import { createProgressBar } from "../progress/progress-bar.js";
|
|
3
|
+
import { createOut } from "../output.js";
|
|
4
|
+
import { withSpinner } from "../spinner.js";
|
|
3
5
|
export async function runExperimentRun(opts) {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const cfg = await loadConfig(opts.config);
|
|
7
|
-
|
|
6
|
+
const out = createOut(opts.verbosity ?? 0);
|
|
7
|
+
out.section(`๐งช Experiment: "${opts.name}"`);
|
|
8
|
+
const cfg = await withSpinner("Loading config", () => loadConfig(opts.config));
|
|
9
|
+
out.info(`Loading eval dataset from "${opts.dataset}"...`);
|
|
8
10
|
const dataset = await loadEvalDataset(opts.dataset);
|
|
9
|
-
await cfg.vectorStore.initialize();
|
|
10
|
-
|
|
11
|
+
await withSpinner("Initializing vector store", () => cfg.vectorStore.initialize());
|
|
12
|
+
out.info("Running evaluation...");
|
|
11
13
|
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
12
14
|
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
13
15
|
let evalResult, perQueryRrScores;
|
|
@@ -17,10 +19,11 @@ export async function runExperimentRun(opts) {
|
|
|
17
19
|
finally {
|
|
18
20
|
evalBar.stop();
|
|
19
21
|
}
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
out.divider();
|
|
23
|
+
out.info(` MRR: ${evalResult.mrr.toFixed(4)}`);
|
|
24
|
+
out.info(` Precision@5: ${(evalResult.precisionAt5 * 100).toFixed(1)}%`);
|
|
25
|
+
out.info(` Recall@10: ${(evalResult.recallAt10 * 100).toFixed(1)}%`);
|
|
26
|
+
out.info(` HitRate@5: ${(evalResult.hitRateAt5 * 100).toFixed(1)}%`);
|
|
24
27
|
const db = new VirageDb(defaultVirageDb());
|
|
25
28
|
try {
|
|
26
29
|
const store = new ExperimentStore(db);
|
|
@@ -33,13 +36,14 @@ export async function runExperimentRun(opts) {
|
|
|
33
36
|
perQueryRrScores,
|
|
34
37
|
};
|
|
35
38
|
const runId = await store.save(run);
|
|
36
|
-
|
|
39
|
+
out.success(`Experiment "${runId}" saved to virage.db`);
|
|
37
40
|
}
|
|
38
41
|
finally {
|
|
39
42
|
db.close();
|
|
40
43
|
}
|
|
41
44
|
}
|
|
42
|
-
export async function runExperimentList() {
|
|
45
|
+
export async function runExperimentList(verbosity = 0) {
|
|
46
|
+
const out = createOut(verbosity);
|
|
43
47
|
const db = new VirageDb(defaultVirageDb());
|
|
44
48
|
let runs;
|
|
45
49
|
try {
|
|
@@ -50,66 +54,64 @@ export async function runExperimentList() {
|
|
|
50
54
|
db.close();
|
|
51
55
|
}
|
|
52
56
|
if (runs.length === 0) {
|
|
53
|
-
|
|
57
|
+
out.info("No experiment runs found. Use `virage experiment run` to create one.");
|
|
54
58
|
return;
|
|
55
59
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
console.log("โ".repeat(80));
|
|
60
|
-
// newest-first
|
|
60
|
+
out.section("๐ Experiment Runs");
|
|
61
|
+
out.info(` ${"ID".padEnd(35)} ${"NAME".padEnd(20)} ${"TIMESTAMP".padEnd(20)} MRR`);
|
|
62
|
+
out.divider("โ", 80);
|
|
61
63
|
const sorted = [...runs].sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
62
64
|
for (const run of sorted) {
|
|
63
65
|
const ts = run.timestamp.slice(0, 19).replace("T", " ");
|
|
64
|
-
|
|
66
|
+
out.info(` ${run.id.padEnd(35)} ${run.name.padEnd(20)} ${ts.padEnd(20)} ${run.evalResult.mrr.toFixed(4)}`);
|
|
65
67
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
+
out.divider("โ", 80);
|
|
69
|
+
out.dim(` ${runs.length} run(s) total`);
|
|
68
70
|
}
|
|
69
71
|
export async function runExperimentCompare(opts) {
|
|
72
|
+
const out = createOut(opts.verbosity ?? 0);
|
|
70
73
|
const db = new VirageDb(defaultVirageDb());
|
|
71
74
|
let baseline, candidate;
|
|
72
75
|
try {
|
|
73
76
|
const store = new ExperimentStore(db);
|
|
74
|
-
|
|
77
|
+
out.info("Loading experiment runs...");
|
|
75
78
|
baseline = await store.load(opts.baseline);
|
|
76
79
|
candidate = await store.load(opts.candidate);
|
|
77
80
|
}
|
|
78
81
|
finally {
|
|
79
82
|
db.close();
|
|
80
83
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
console.log(` Candidate : ${candidate.id} (MRR: ${candidate.evalResult.mrr.toFixed(4)})`);
|
|
84
|
+
out.section("๐ Comparing experiments");
|
|
85
|
+
out.info(` Baseline : ${baseline.id} (MRR: ${baseline.evalResult.mrr.toFixed(4)})`);
|
|
86
|
+
out.info(` Candidate : ${candidate.id} (MRR: ${candidate.evalResult.mrr.toFixed(4)})`);
|
|
85
87
|
if (!baseline.perQueryRrScores || !candidate.perQueryRrScores) {
|
|
86
88
|
const delta = candidate.evalResult.mrr - baseline.evalResult.mrr;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
89
|
+
out.divider();
|
|
90
|
+
out.info(` MRR delta : ${delta >= 0 ? "+" : ""}${delta.toFixed(4)}`);
|
|
91
|
+
out.dim(" Per-query scores unavailable. Re-run experiments to get bootstrap test.");
|
|
92
|
+
out.divider();
|
|
91
93
|
return;
|
|
92
94
|
}
|
|
93
95
|
if (baseline.perQueryRrScores.length !== candidate.perQueryRrScores.length) {
|
|
94
|
-
|
|
96
|
+
out.warn(`Query count mismatch (${baseline.perQueryRrScores.length} vs ` +
|
|
95
97
|
`${candidate.perQueryRrScores.length}). Using shorter set.`);
|
|
96
98
|
}
|
|
97
99
|
const n = Math.min(baseline.perQueryRrScores.length, candidate.perQueryRrScores.length);
|
|
98
100
|
if (n < 2) {
|
|
99
|
-
|
|
101
|
+
out.warn("Cannot run statistical test: need at least 2 queries.");
|
|
100
102
|
return;
|
|
101
103
|
}
|
|
102
104
|
const stat = bootstrapPairedTest(baseline.perQueryRrScores.slice(0, n), candidate.perQueryRrScores.slice(0, n));
|
|
103
|
-
const
|
|
105
|
+
const verdict = stat.recommendation === "accept"
|
|
104
106
|
? "โ
"
|
|
105
107
|
: stat.recommendation === "reject"
|
|
106
108
|
? "โ"
|
|
107
109
|
: "โ ๏ธ ";
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
110
|
+
out.divider();
|
|
111
|
+
out.info(` MRR delta : ${stat.mrrDelta >= 0 ? "+" : ""}${stat.mrrDelta.toFixed(4)}`);
|
|
112
|
+
out.info(` p-value : ${stat.pValue.toFixed(4)}`);
|
|
113
|
+
out.info(` 95% CI : [${stat.confidenceInterval95[0].toFixed(4)}, ${stat.confidenceInterval95[1].toFixed(4)}]`);
|
|
114
|
+
out.info(` Verdict : ${verdict} ${stat.recommendation.toUpperCase()}`);
|
|
115
|
+
out.divider();
|
|
114
116
|
}
|
|
115
117
|
//# sourceMappingURL=experiment.js.map
|