@vivantel/virage-cli 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/virage.d.ts +3 -0
- package/dist/bin/virage.d.ts.map +1 -0
- package/dist/bin/virage.js +366 -0
- package/dist/bin/virage.js.map +1 -0
- package/dist/cli/benchmark.d.ts +6 -0
- package/dist/cli/benchmark.d.ts.map +1 -0
- package/dist/cli/benchmark.js +26 -0
- package/dist/cli/benchmark.js.map +1 -0
- package/dist/cli/chunks-report.d.ts +2 -0
- package/dist/cli/chunks-report.d.ts.map +1 -0
- package/dist/cli/chunks-report.js +74 -0
- package/dist/cli/chunks-report.js.map +1 -0
- package/dist/cli/dashboard.d.ts +7 -0
- package/dist/cli/dashboard.d.ts.map +1 -0
- package/dist/cli/dashboard.js +164 -0
- package/dist/cli/dashboard.js.map +1 -0
- package/dist/cli/evaluate.d.ts +9 -0
- package/dist/cli/evaluate.d.ts.map +1 -0
- package/dist/cli/evaluate.js +64 -0
- package/dist/cli/evaluate.js.map +1 -0
- package/dist/cli/experiment.d.ts +13 -0
- package/dist/cli/experiment.d.ts.map +1 -0
- package/dist/cli/experiment.js +95 -0
- package/dist/cli/experiment.js.map +1 -0
- package/dist/cli/file-detect.d.ts +11 -0
- package/dist/cli/file-detect.d.ts.map +1 -0
- package/dist/cli/file-detect.js +46 -0
- package/dist/cli/file-detect.js.map +1 -0
- package/dist/cli/init.d.ts +2 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +381 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/report.d.ts +2 -0
- package/dist/cli/report.d.ts.map +1 -0
- package/dist/cli/report.js +96 -0
- package/dist/cli/report.js.map +1 -0
- package/dist/cli/store-cmd.d.ts +10 -0
- package/dist/cli/store-cmd.d.ts.map +1 -0
- package/dist/cli/store-cmd.js +50 -0
- package/dist/cli/store-cmd.js.map +1 -0
- package/dist/cli/validate.d.ts +2 -0
- package/dist/cli/validate.d.ts.map +1 -0
- package/dist/cli/validate.js +72 -0
- package/dist/cli/validate.js.map +1 -0
- package/dist/cli/viz.d.ts +7 -0
- package/dist/cli/viz.d.ts.map +1 -0
- package/dist/cli/viz.js +163 -0
- package/dist/cli/viz.js.map +1 -0
- package/dist/dashboard-ui/assets/index-BD53Fj5e.js +40 -0
- package/dist/dashboard-ui/assets/index-Boh3NVMF.css +1 -0
- package/dist/dashboard-ui/index.html +13 -0
- package/dist/logger/consola-logger.d.ts +18 -0
- package/dist/logger/consola-logger.d.ts.map +1 -0
- package/dist/logger/consola-logger.js +76 -0
- package/dist/logger/consola-logger.js.map +1 -0
- package/dist/logger/index.d.ts +4 -0
- package/dist/logger/index.d.ts.map +1 -0
- package/dist/logger/index.js +6 -0
- package/dist/logger/index.js.map +1 -0
- package/dist/progress/progress-bar.d.ts +6 -0
- package/dist/progress/progress-bar.d.ts.map +1 -0
- package/dist/progress/progress-bar.js +23 -0
- package/dist/progress/progress-bar.js.map +1 -0
- package/package.json +75 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { createServer } from "http";
|
|
2
|
+
import { readFile } from "fs/promises";
|
|
3
|
+
import { existsSync } from "fs";
|
|
4
|
+
import { join, extname } from "path";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { EmbeddingsDb } from "@vivantel/virage-core";
|
|
7
|
+
const MIME = {
|
|
8
|
+
".html": "text/html; charset=utf-8",
|
|
9
|
+
".js": "application/javascript",
|
|
10
|
+
".css": "text/css",
|
|
11
|
+
".svg": "image/svg+xml",
|
|
12
|
+
".ico": "image/x-icon",
|
|
13
|
+
};
|
|
14
|
+
// Resolve the compiled dashboard-ui/ directory shipped alongside this package.
|
|
15
|
+
// At runtime the compiled CLI lives at dist/cli/dashboard.js; dashboard-ui is at dist/dashboard-ui/.
|
|
16
|
+
const THIS_DIR = fileURLToPath(new URL(".", import.meta.url));
|
|
17
|
+
const UI_DIR = join(THIS_DIR, "..", "dashboard-ui");
|
|
18
|
+
const HAS_UI = existsSync(join(UI_DIR, "index.html"));
|
|
19
|
+
async function serveStatic(urlPath, res) {
|
|
20
|
+
if (!HAS_UI)
|
|
21
|
+
return false;
|
|
22
|
+
const normalized = urlPath === "/" || urlPath === "" ? "/index.html" : urlPath;
|
|
23
|
+
// Only serve known static assets; fall back to index.html for SPA routing
|
|
24
|
+
const filePath = join(UI_DIR, normalized);
|
|
25
|
+
const ext = extname(filePath).toLowerCase();
|
|
26
|
+
try {
|
|
27
|
+
const content = await readFile(filePath);
|
|
28
|
+
res.setHeader("Content-Type", MIME[ext] ?? "application/octet-stream");
|
|
29
|
+
res.end(content);
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// Try index.html for SPA client-side routing (non-asset paths)
|
|
34
|
+
if (!normalized.startsWith("/assets/") && ext === ".html") {
|
|
35
|
+
try {
|
|
36
|
+
const html = await readFile(join(UI_DIR, "index.html"));
|
|
37
|
+
res.setHeader("Content-Type", "text/html; charset=utf-8");
|
|
38
|
+
res.end(html);
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export async function runDashboard(opts) {
|
|
49
|
+
if (!HAS_UI) {
|
|
50
|
+
console.warn("ā ļø Dashboard UI not found. Run `npm run build -w @vivantel/virage-dashboard` first.");
|
|
51
|
+
}
|
|
52
|
+
const server = createServer(async (req, res) => {
|
|
53
|
+
const url = req.url ?? "/";
|
|
54
|
+
res.setHeader("Content-Type", "application/json");
|
|
55
|
+
try {
|
|
56
|
+
if (url === "/api/status") {
|
|
57
|
+
const status = await getStatus(opts.chunksFile, opts.dbPath);
|
|
58
|
+
res.end(JSON.stringify(status));
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
if (url === "/api/chunks") {
|
|
62
|
+
const chunks = await getChunksHistogram(opts.chunksFile);
|
|
63
|
+
res.end(JSON.stringify(chunks));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (url === "/api/embeddings/anomalies") {
|
|
67
|
+
const anomalies = await getAnomalies(opts.dbPath);
|
|
68
|
+
res.end(JSON.stringify(anomalies));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
// Serve React app static assets
|
|
72
|
+
const served = await serveStatic(url, res);
|
|
73
|
+
if (!served) {
|
|
74
|
+
res.statusCode = 404;
|
|
75
|
+
res.end(JSON.stringify({ error: "Not found" }));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch (err) {
|
|
79
|
+
res.statusCode = 500;
|
|
80
|
+
res.end(JSON.stringify({ error: err.message }));
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
server.listen(opts.port, () => {
|
|
84
|
+
console.log(`š RAG Dashboard running at http://localhost:${opts.port}`);
|
|
85
|
+
console.log(" Press Ctrl+C to stop");
|
|
86
|
+
});
|
|
87
|
+
// Keep process alive
|
|
88
|
+
await new Promise((resolve) => {
|
|
89
|
+
process.on("SIGINT", () => {
|
|
90
|
+
server.close();
|
|
91
|
+
resolve();
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
async function getStatus(chunksFile, dbPath) {
|
|
96
|
+
let totalChunks = 0;
|
|
97
|
+
let totalEmbeddings = 0;
|
|
98
|
+
try {
|
|
99
|
+
const chunks = JSON.parse(await readFile(chunksFile, "utf-8"));
|
|
100
|
+
totalChunks = Array.isArray(chunks)
|
|
101
|
+
? chunks.length
|
|
102
|
+
: (chunks.chunks?.length ?? 0);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
/* file may not exist yet */
|
|
106
|
+
}
|
|
107
|
+
try {
|
|
108
|
+
const db = new EmbeddingsDb(dbPath);
|
|
109
|
+
totalEmbeddings = db.getAll().length;
|
|
110
|
+
db.close();
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
/* db may not exist yet */
|
|
114
|
+
}
|
|
115
|
+
const memoryMB = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
|
|
116
|
+
return { totalChunks, totalEmbeddings, memoryMB };
|
|
117
|
+
}
|
|
118
|
+
async function getChunksHistogram(chunksFile) {
|
|
119
|
+
try {
|
|
120
|
+
const raw = JSON.parse(await readFile(chunksFile, "utf-8"));
|
|
121
|
+
const chunks = (Array.isArray(raw) ? raw : (raw.chunks ?? []));
|
|
122
|
+
const sizes = chunks.map((c) => c.content?.length ?? 0);
|
|
123
|
+
const buckets = [
|
|
124
|
+
{ label: "< 200 chars", min: 0, max: 200 },
|
|
125
|
+
{ label: "200ā500 chars", min: 200, max: 500 },
|
|
126
|
+
{ label: "500ā1k chars", min: 500, max: 1000 },
|
|
127
|
+
{ label: "1kā2k chars", min: 1000, max: 2000 },
|
|
128
|
+
{ label: "> 2k chars", min: 2000, max: Infinity },
|
|
129
|
+
];
|
|
130
|
+
const histogram = buckets.map((b) => ({
|
|
131
|
+
label: b.label,
|
|
132
|
+
count: sizes.filter((s) => s >= b.min && s < b.max).length,
|
|
133
|
+
}));
|
|
134
|
+
return { histogram };
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return { histogram: [] };
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function getAnomalies(dbPath) {
|
|
141
|
+
try {
|
|
142
|
+
const db = new EmbeddingsDb(dbPath);
|
|
143
|
+
const chunks = db.getAll();
|
|
144
|
+
db.close();
|
|
145
|
+
if (chunks.length === 0)
|
|
146
|
+
return { anomalies: [] };
|
|
147
|
+
const norms = chunks.map((c) => Math.sqrt((c.embedding ?? []).reduce((s, v) => s + v * v, 0)));
|
|
148
|
+
const mean = norms.reduce((s, v) => s + v, 0) / norms.length;
|
|
149
|
+
const std = Math.sqrt(norms.reduce((s, v) => s + (v - mean) ** 2, 0) / norms.length);
|
|
150
|
+
const anomalies = chunks
|
|
151
|
+
.map((c, i) => ({
|
|
152
|
+
sourceFile: c.sourceFile ?? "unknown",
|
|
153
|
+
zscore: std > 0 ? Math.abs(norms[i] - mean) / std : 0,
|
|
154
|
+
preview: (c.content ?? "").slice(0, 60),
|
|
155
|
+
}))
|
|
156
|
+
.filter((a) => a.zscore > 2.5)
|
|
157
|
+
.sort((a, b) => b.zscore - a.zscore);
|
|
158
|
+
return { anomalies };
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return { anomalies: [] };
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=dashboard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dashboard.js","sourceRoot":"","sources":["../../src/cli/dashboard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAA6C,MAAM,MAAM,CAAC;AAC/E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAQrD,MAAM,IAAI,GAA2B;IACnC,OAAO,EAAE,0BAA0B;IACnC,KAAK,EAAE,wBAAwB;IAC/B,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,eAAe;IACvB,MAAM,EAAE,cAAc;CACvB,CAAC;AAEF,+EAA+E;AAC/E,qGAAqG;AACrG,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;AACpD,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;AAEtD,KAAK,UAAU,WAAW,CAAC,OAAe,EAAE,GAAmB;IAC7D,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAE1B,MAAM,UAAU,GAAG,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/E,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,0BAA0B,CAAC,CAAC;QACvE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;QAC/D,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;gBACxD,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,0BAA0B,CAAC,CAAC;gBAC1D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACvD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CACV,sFAAsF,CACvF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CACzB,KAAK,EAAE,GAAoB,EAAE,GAAmB,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC;QAC3B,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;QAElD,IAAI,CAAC;YACH,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,IAAI,GAAG,KAAK,2BAA2B,EAAE,CAAC;gBACxC,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAClD,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;gBACnC,OAAO;YACT,CAAC;YAED,gCAAgC;YAChC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YAC3C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;gBACrB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC;YACrB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE;QAC5B,OAAO,CAAC,GAAG,CAAC,gDAAgD,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YACxB,MAAM,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,UAAkB,EAAE,MAAc;IACzD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAY,CAAC;QAC1E,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,MAAM;YACf,CAAC,CAAC,CAAE,MAAiC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,4BAA4B;IAC9B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QACpC,eAAe,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC;QACrC,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;IAC5B,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC;IAE1E,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,UAAkB;IAClD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAY,CAAC;QACvE,MAAM,MAAM,GAAG,CACb,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAA8B,CAAC,MAAM,IAAI,EAAE,CAAC,CAC5C,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAExD,MAAM,OAAO,GAAG;YACd,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE;YAC1C,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;YAC9C,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE;YAC9C,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE;YAC9C,EAAE,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE;SAClD,CAAC;QAEF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM;SAC3D,CAAC,CAAC,CAAC;QAEJ,OAAO,EAAE,SAAS,EAAE,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAc;IACxC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,KAAK,EAAE,CAAC;QAEX,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAElD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7B,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACnE,CACF,CAAC;QACF,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CACnB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAC9D,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACd,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,SAAS;YACrC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACrD,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACxC,CAAC,CAAC;aACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC;aAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAEvC,OAAO,EAAE,SAAS,EAAE,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,EAAE,EAAE,OAAO,CAAC;CACb;AAkBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CA4DtE"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { loadConfig, loadEvalDataset, EvalRunner, ExperimentStore, makeRunId, } from "@vivantel/virage-core";
|
|
2
|
+
import { createProgressBar } from "../progress/progress-bar.js";
|
|
3
|
+
function formatPercent(n) {
|
|
4
|
+
return `${(n * 100).toFixed(1)}%`;
|
|
5
|
+
}
|
|
6
|
+
function printEvalResult(result) {
|
|
7
|
+
console.log("\nš Retrieval Evaluation Results");
|
|
8
|
+
console.log("ā".repeat(40));
|
|
9
|
+
console.log(` Queries evaluated : ${result.queriesEvaluated}`);
|
|
10
|
+
console.log(` Precision@5 : ${formatPercent(result.precisionAt5)}`);
|
|
11
|
+
console.log(` Precision@10 : ${formatPercent(result.precisionAt10)}`);
|
|
12
|
+
console.log(` Recall@10 : ${formatPercent(result.recallAt10)}`);
|
|
13
|
+
console.log(` MRR : ${result.mrr.toFixed(4)}`);
|
|
14
|
+
console.log(` HitRate@5 : ${formatPercent(result.hitRateAt5)}`);
|
|
15
|
+
console.log("ā".repeat(40));
|
|
16
|
+
}
|
|
17
|
+
export async function runEvaluate(opts) {
|
|
18
|
+
console.log("š Loading config...");
|
|
19
|
+
const cfg = await loadConfig(opts.config);
|
|
20
|
+
console.log(`š Loading eval dataset from "${opts.dataset}"...`);
|
|
21
|
+
const dataset = await loadEvalDataset(opts.dataset);
|
|
22
|
+
console.log(` Found ${dataset.queries.length} queries`);
|
|
23
|
+
await cfg.vectorStore.initialize();
|
|
24
|
+
console.log("š Running retrieval evaluation...");
|
|
25
|
+
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
26
|
+
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
27
|
+
let evalResult, perQueryRrScores;
|
|
28
|
+
try {
|
|
29
|
+
({ evalResult, perQueryRrScores } = await runner.run((done, total) => evalBar.update(done < total ? done : total)));
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
evalBar.stop();
|
|
33
|
+
}
|
|
34
|
+
printEvalResult(evalResult);
|
|
35
|
+
if (opts.withLlmJudge) {
|
|
36
|
+
console.log("\nāļø LLM-as-judge (RAGAS) requires a judge configured in config.");
|
|
37
|
+
console.log(" Add a judge to your pipeline config to enable RAGAS metrics.");
|
|
38
|
+
}
|
|
39
|
+
// Save to experiment store
|
|
40
|
+
const store = new ExperimentStore();
|
|
41
|
+
const run = {
|
|
42
|
+
id: makeRunId("eval"),
|
|
43
|
+
name: "eval",
|
|
44
|
+
timestamp: new Date().toISOString(),
|
|
45
|
+
config: { configFile: opts.config, dataset: opts.dataset },
|
|
46
|
+
evalResult,
|
|
47
|
+
perQueryRrScores,
|
|
48
|
+
};
|
|
49
|
+
const savedPath = await store.save(run);
|
|
50
|
+
console.log(`\nš¾ Results saved to: ${savedPath}`);
|
|
51
|
+
// CI quality gate
|
|
52
|
+
if (opts.thresholdMrr !== undefined) {
|
|
53
|
+
const passed = evalResult.mrr >= opts.thresholdMrr;
|
|
54
|
+
if (passed) {
|
|
55
|
+
console.log(`\nā
Quality gate passed: MRR ${evalResult.mrr.toFixed(4)} ā„ ${opts.thresholdMrr}`);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
console.error(`\nā Quality gate FAILED: MRR ${evalResult.mrr.toFixed(4)} < ${opts.thresholdMrr}`);
|
|
59
|
+
if (opts.ci)
|
|
60
|
+
process.exit(1);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=evaluate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,eAAe,EACf,UAAU,EACV,eAAe,EACf,SAAS,GACV,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAWhE,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CAAC,MAAkB;IACzC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC3E,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB;IACrD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE1D,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,IAAI,UAAU,EAAE,gBAAgB,CAAC;IACjC,IAAI,CAAC;QACH,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACnE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC5C,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;IAED,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,mEAAmE,CACpE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,iEAAiE,CAClE,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAkB;QACzB,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;QAC1D,UAAU;QACV,gBAAgB;KACjB,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,0BAA0B,SAAS,EAAE,CAAC,CAAC;IAEnD,kBAAkB;IAClB,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC;QACnD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CACT,gCAAgC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACnF,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CACX,gCAAgC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACnF,CAAC;YACF,IAAI,IAAI,CAAC,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export interface ExperimentRunOptions {
|
|
2
|
+
name: string;
|
|
3
|
+
config: string;
|
|
4
|
+
dataset: string;
|
|
5
|
+
}
|
|
6
|
+
export interface ExperimentCompareOptions {
|
|
7
|
+
baseline: string;
|
|
8
|
+
candidate: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function runExperimentRun(opts: ExperimentRunOptions): Promise<void>;
|
|
11
|
+
export declare function runExperimentList(): Promise<void>;
|
|
12
|
+
export declare function runExperimentCompare(opts: ExperimentCompareOptions): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=experiment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,IAAI,CAAC,CAuCf;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAgCvD;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,wBAAwB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAoEf"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { loadConfig, loadEvalDataset, EvalRunner, ExperimentStore, makeRunId, bootstrapPairedTest, } from "@vivantel/virage-core";
|
|
2
|
+
import { createProgressBar } from "../progress/progress-bar.js";
|
|
3
|
+
export async function runExperimentRun(opts) {
|
|
4
|
+
console.log(`š§Ŗ Experiment: "${opts.name}"`);
|
|
5
|
+
console.log("š Loading config...");
|
|
6
|
+
const cfg = await loadConfig(opts.config);
|
|
7
|
+
console.log(`š Loading eval dataset from "${opts.dataset}"...`);
|
|
8
|
+
const dataset = await loadEvalDataset(opts.dataset);
|
|
9
|
+
await cfg.vectorStore.initialize();
|
|
10
|
+
console.log("š Running evaluation...");
|
|
11
|
+
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
12
|
+
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
13
|
+
let evalResult, perQueryRrScores;
|
|
14
|
+
try {
|
|
15
|
+
({ evalResult, perQueryRrScores } = await runner.run((done, total) => evalBar.update(done < total ? done : total)));
|
|
16
|
+
}
|
|
17
|
+
finally {
|
|
18
|
+
evalBar.stop();
|
|
19
|
+
}
|
|
20
|
+
console.log(`\n MRR: ${evalResult.mrr.toFixed(4)}`);
|
|
21
|
+
console.log(` Precision@5: ${(evalResult.precisionAt5 * 100).toFixed(1)}%`);
|
|
22
|
+
console.log(` Recall@10: ${(evalResult.recallAt10 * 100).toFixed(1)}%`);
|
|
23
|
+
console.log(` HitRate@5: ${(evalResult.hitRateAt5 * 100).toFixed(1)}%`);
|
|
24
|
+
const store = new ExperimentStore();
|
|
25
|
+
const run = {
|
|
26
|
+
id: makeRunId(opts.name),
|
|
27
|
+
name: opts.name,
|
|
28
|
+
timestamp: new Date().toISOString(),
|
|
29
|
+
config: { configFile: opts.config, dataset: opts.dataset },
|
|
30
|
+
evalResult,
|
|
31
|
+
perQueryRrScores,
|
|
32
|
+
};
|
|
33
|
+
const savedPath = await store.save(run);
|
|
34
|
+
console.log(`\nš¾ Experiment "${run.id}" saved to: ${savedPath}`);
|
|
35
|
+
}
|
|
36
|
+
export async function runExperimentList() {
|
|
37
|
+
const store = new ExperimentStore();
|
|
38
|
+
const runs = await store.list();
|
|
39
|
+
if (runs.length === 0) {
|
|
40
|
+
console.log("No experiment runs found. Use `virage experiment run` to create one.");
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
console.log("\nš Experiment Runs");
|
|
44
|
+
console.log("ā".repeat(80));
|
|
45
|
+
console.log(` ${"ID".padEnd(35)} ${"NAME".padEnd(20)} ${"TIMESTAMP".padEnd(20)} MRR`);
|
|
46
|
+
console.log("ā".repeat(80));
|
|
47
|
+
// newest-first
|
|
48
|
+
const sorted = [...runs].sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
|
|
49
|
+
for (const run of sorted) {
|
|
50
|
+
const ts = run.timestamp.slice(0, 19).replace("T", " ");
|
|
51
|
+
console.log(` ${run.id.padEnd(35)} ${run.name.padEnd(20)} ${ts.padEnd(20)} ${run.evalResult.mrr.toFixed(4)}`);
|
|
52
|
+
}
|
|
53
|
+
console.log("ā".repeat(80));
|
|
54
|
+
console.log(` ${runs.length} run(s) total`);
|
|
55
|
+
}
|
|
56
|
+
export async function runExperimentCompare(opts) {
|
|
57
|
+
const store = new ExperimentStore();
|
|
58
|
+
console.log(`š Loading experiment runs...`);
|
|
59
|
+
const baseline = await store.load(opts.baseline);
|
|
60
|
+
const candidate = await store.load(opts.candidate);
|
|
61
|
+
console.log(`\nš Comparing experiments`);
|
|
62
|
+
console.log("ā".repeat(50));
|
|
63
|
+
console.log(` Baseline : ${baseline.id} (MRR: ${baseline.evalResult.mrr.toFixed(4)})`);
|
|
64
|
+
console.log(` Candidate : ${candidate.id} (MRR: ${candidate.evalResult.mrr.toFixed(4)})`);
|
|
65
|
+
if (!baseline.perQueryRrScores || !candidate.perQueryRrScores) {
|
|
66
|
+
const delta = candidate.evalResult.mrr - baseline.evalResult.mrr;
|
|
67
|
+
console.log("ā".repeat(50));
|
|
68
|
+
console.log(` MRR delta : ${delta >= 0 ? "+" : ""}${delta.toFixed(4)}`);
|
|
69
|
+
console.log(` ā¹ļø Per-query scores unavailable. Re-run experiments to get bootstrap test.`);
|
|
70
|
+
console.log("ā".repeat(50));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (baseline.perQueryRrScores.length !== candidate.perQueryRrScores.length) {
|
|
74
|
+
console.warn(`ā ļø Query count mismatch (${baseline.perQueryRrScores.length} vs ` +
|
|
75
|
+
`${candidate.perQueryRrScores.length}). Using shorter set.`);
|
|
76
|
+
}
|
|
77
|
+
const n = Math.min(baseline.perQueryRrScores.length, candidate.perQueryRrScores.length);
|
|
78
|
+
if (n < 2) {
|
|
79
|
+
console.log("\nā ļø Cannot run statistical test: need at least 2 queries.");
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const stat = bootstrapPairedTest(baseline.perQueryRrScores.slice(0, n), candidate.perQueryRrScores.slice(0, n));
|
|
83
|
+
const verdictEmoji = stat.recommendation === "accept"
|
|
84
|
+
? "ā
"
|
|
85
|
+
: stat.recommendation === "reject"
|
|
86
|
+
? "ā"
|
|
87
|
+
: "ā ļø ";
|
|
88
|
+
console.log("ā".repeat(50));
|
|
89
|
+
console.log(` MRR delta : ${stat.mrrDelta >= 0 ? "+" : ""}${stat.mrrDelta.toFixed(4)}`);
|
|
90
|
+
console.log(` p-value : ${stat.pValue.toFixed(4)}`);
|
|
91
|
+
console.log(` 95% CI : [${stat.confidenceInterval95[0].toFixed(4)}, ${stat.confidenceInterval95[1].toFixed(4)}]`);
|
|
92
|
+
console.log(` Verdict : ${verdictEmoji} ${stat.recommendation.toUpperCase()}`);
|
|
93
|
+
console.log("ā".repeat(50));
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=experiment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,UAAU,EACV,eAAe,EACf,UAAU,EACV,eAAe,EACf,SAAS,EACT,mBAAmB,GACpB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAchE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAA0B;IAE1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpD,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,IAAI,UAAU,EAAE,gBAAgB,CAAC;IACjC,IAAI,CAAC;QACH,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACnE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC5C,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,IAAI,EAAE,CAAC;IACjB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,UAAU,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAEzE,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAkB;QACzB,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;QAC1D,UAAU;QACV,gBAAgB;KACjB,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,CAAC,EAAE,eAAe,SAAS,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IAEhC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,sEAAsE,CACvE,CAAC;QACF,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAC1E,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,eAAe;IACf,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CACT,KAAK,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,eAAe,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IAEpC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEnD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,iBAAiB,QAAQ,CAAC,EAAE,UAAU,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC5E,CAAC;IACF,OAAO,CAAC,GAAG,CACT,iBAAiB,SAAS,CAAC,EAAE,UAAU,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC9E,CAAC;IAEF,IAAI,CAAC,QAAQ,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,+EAA+E,CAChF,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,KAAK,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC3E,OAAO,CAAC,IAAI,CACV,6BAA6B,QAAQ,CAAC,gBAAgB,CAAC,MAAM,MAAM;YACjE,GAAG,SAAS,CAAC,gBAAgB,CAAC,MAAM,uBAAuB,CAC9D,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAChC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAClC,CAAC;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QAC3E,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,mBAAmB,CAC9B,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EACrC,SAAS,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CACvC,CAAC;IAEF,MAAM,YAAY,GAChB,IAAI,CAAC,cAAc,KAAK,QAAQ;QAC9B,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,IAAI,CAAC,cAAc,KAAK,QAAQ;YAChC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,KAAK,CAAC;IAEd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,kBAAkB,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CACT,mBAAmB,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1G,CAAC;IACF,OAAO,CAAC,GAAG,CACT,kBAAkB,YAAY,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EAAE,CACtE,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { IGNORED_DIRS } from "@vivantel/virage-core";
|
|
2
|
+
export { IGNORED_DIRS };
|
|
3
|
+
export interface ExtGroup {
|
|
4
|
+
exts: string[];
|
|
5
|
+
strategyFn: string;
|
|
6
|
+
name: string;
|
|
7
|
+
}
|
|
8
|
+
export declare const EXT_GROUPS: ExtGroup[];
|
|
9
|
+
export declare function collectExtensions(dir: string, found: Set<string>): Promise<void>;
|
|
10
|
+
export declare function detectFileExtensions(cwd: string): Promise<ExtGroup[]>;
|
|
11
|
+
//# sourceMappingURL=file-detect.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-detect.d.ts","sourceRoot":"","sources":["../../src/cli/file-detect.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,YAAY,EAAE,CAAC;AAExB,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,eAAO,MAAM,UAAU,EAAE,QAAQ,EAchC,CAAC;AAEF,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,GACjB,OAAO,CAAC,IAAI,CAAC,CAiBf;AAED,wBAAsB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAI3E"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { readdir } from "fs/promises";
|
|
2
|
+
import { extname, join } from "path";
|
|
3
|
+
import { IGNORED_DIRS } from "@vivantel/virage-core";
|
|
4
|
+
export { IGNORED_DIRS };
|
|
5
|
+
export const EXT_GROUPS = [
|
|
6
|
+
{
|
|
7
|
+
exts: [".md", ".mdx"],
|
|
8
|
+
strategyFn: "markdownHeadersStrategy",
|
|
9
|
+
name: "markdown",
|
|
10
|
+
},
|
|
11
|
+
{ exts: [".ts", ".tsx"], strategyFn: "tokenStrategy", name: "typescript" },
|
|
12
|
+
{ exts: [".js", ".jsx"], strategyFn: "tokenStrategy", name: "javascript" },
|
|
13
|
+
{ exts: [".py"], strategyFn: "tokenStrategy", name: "python" },
|
|
14
|
+
{ exts: [".go"], strategyFn: "tokenStrategy", name: "go" },
|
|
15
|
+
{ exts: [".cs"], strategyFn: "tokenStrategy", name: "csharp" },
|
|
16
|
+
{ exts: [".java"], strategyFn: "tokenStrategy", name: "java" },
|
|
17
|
+
{ exts: [".yaml", ".yml"], strategyFn: "wholeFileStrategy", name: "yaml" },
|
|
18
|
+
{ exts: [".txt"], strategyFn: "semanticStrategy", name: "text" },
|
|
19
|
+
];
|
|
20
|
+
export async function collectExtensions(dir, found) {
|
|
21
|
+
let entries;
|
|
22
|
+
try {
|
|
23
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
for (const entry of entries) {
|
|
29
|
+
if (entry.isDirectory()) {
|
|
30
|
+
if (!IGNORED_DIRS.has(entry.name) && !entry.name.startsWith(".")) {
|
|
31
|
+
await collectExtensions(join(dir, entry.name), found);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
const ext = extname(entry.name).toLowerCase();
|
|
36
|
+
if (ext)
|
|
37
|
+
found.add(ext);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export async function detectFileExtensions(cwd) {
|
|
42
|
+
const found = new Set();
|
|
43
|
+
await collectExtensions(cwd, found);
|
|
44
|
+
return EXT_GROUPS.filter((g) => g.exts.some((e) => found.has(e)));
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=file-detect.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-detect.js","sourceRoot":"","sources":["../../src/cli/file-detect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,EAAE,YAAY,EAAE,CAAC;AAQxB,MAAM,CAAC,MAAM,UAAU,GAAe;IACpC;QACE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,yBAAyB;QACrC,IAAI,EAAE,UAAU;KACjB;IACD,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE;IAC1D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE;CACjE,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,KAAkB;IAElB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IACpD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAkPA,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAmP7C"}
|