eqho-eval 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +552 -0
- package/dist/cli/auth-store.d.ts +5 -0
- package/dist/cli/auth-store.d.ts.map +1 -0
- package/dist/cli/auth-store.js +39 -0
- package/dist/cli/auth-store.js.map +1 -0
- package/dist/cli/banner.d.ts +3 -0
- package/dist/cli/banner.d.ts.map +1 -0
- package/dist/cli/banner.js +38 -0
- package/dist/cli/banner.js.map +1 -0
- package/dist/cli/commands/action-eval.d.ts +3 -0
- package/dist/cli/commands/action-eval.d.ts.map +1 -0
- package/dist/cli/commands/action-eval.js +133 -0
- package/dist/cli/commands/action-eval.js.map +1 -0
- package/dist/cli/commands/auth.d.ts +3 -0
- package/dist/cli/commands/auth.d.ts.map +1 -0
- package/dist/cli/commands/auth.js +156 -0
- package/dist/cli/commands/auth.js.map +1 -0
- package/dist/cli/commands/cache.d.ts +3 -0
- package/dist/cli/commands/cache.d.ts.map +1 -0
- package/dist/cli/commands/cache.js +43 -0
- package/dist/cli/commands/cache.js.map +1 -0
- package/dist/cli/commands/ci.d.ts +3 -0
- package/dist/cli/commands/ci.d.ts.map +1 -0
- package/dist/cli/commands/ci.js +124 -0
- package/dist/cli/commands/ci.js.map +1 -0
- package/dist/cli/commands/conversations.d.ts +3 -0
- package/dist/cli/commands/conversations.d.ts.map +1 -0
- package/dist/cli/commands/conversations.js +89 -0
- package/dist/cli/commands/conversations.js.map +1 -0
- package/dist/cli/commands/diff.d.ts +3 -0
- package/dist/cli/commands/diff.d.ts.map +1 -0
- package/dist/cli/commands/diff.js +122 -0
- package/dist/cli/commands/diff.js.map +1 -0
- package/dist/cli/commands/doctor.d.ts +11 -0
- package/dist/cli/commands/doctor.d.ts.map +1 -0
- package/dist/cli/commands/doctor.js +308 -0
- package/dist/cli/commands/doctor.js.map +1 -0
- package/dist/cli/commands/eval.d.ts +3 -0
- package/dist/cli/commands/eval.d.ts.map +1 -0
- package/dist/cli/commands/eval.js +101 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/commands/init.d.ts +3 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +182 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/list.d.ts +3 -0
- package/dist/cli/commands/list.d.ts.map +1 -0
- package/dist/cli/commands/list.js +80 -0
- package/dist/cli/commands/list.js.map +1 -0
- package/dist/cli/commands/mentions.d.ts +3 -0
- package/dist/cli/commands/mentions.d.ts.map +1 -0
- package/dist/cli/commands/mentions.js +125 -0
- package/dist/cli/commands/mentions.js.map +1 -0
- package/dist/cli/commands/org.d.ts +3 -0
- package/dist/cli/commands/org.d.ts.map +1 -0
- package/dist/cli/commands/org.js +196 -0
- package/dist/cli/commands/org.js.map +1 -0
- package/dist/cli/commands/postcall-eval.d.ts +3 -0
- package/dist/cli/commands/postcall-eval.d.ts.map +1 -0
- package/dist/cli/commands/postcall-eval.js +188 -0
- package/dist/cli/commands/postcall-eval.js.map +1 -0
- package/dist/cli/commands/render.d.ts +3 -0
- package/dist/cli/commands/render.d.ts.map +1 -0
- package/dist/cli/commands/render.js +223 -0
- package/dist/cli/commands/render.js.map +1 -0
- package/dist/cli/commands/results.d.ts +3 -0
- package/dist/cli/commands/results.d.ts.map +1 -0
- package/dist/cli/commands/results.js +128 -0
- package/dist/cli/commands/results.js.map +1 -0
- package/dist/cli/commands/scenarios.d.ts +3 -0
- package/dist/cli/commands/scenarios.d.ts.map +1 -0
- package/dist/cli/commands/scenarios.js +57 -0
- package/dist/cli/commands/scenarios.js.map +1 -0
- package/dist/cli/commands/start.d.ts +3 -0
- package/dist/cli/commands/start.d.ts.map +1 -0
- package/dist/cli/commands/start.js +260 -0
- package/dist/cli/commands/start.js.map +1 -0
- package/dist/cli/commands/status.d.ts +3 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +133 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/sync.d.ts +3 -0
- package/dist/cli/commands/sync.d.ts.map +1 -0
- package/dist/cli/commands/sync.js +80 -0
- package/dist/cli/commands/sync.js.map +1 -0
- package/dist/cli/commands/view.d.ts +3 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +29 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/error-handler.d.ts +8 -0
- package/dist/cli/error-handler.d.ts.map +1 -0
- package/dist/cli/error-handler.js +133 -0
- package/dist/cli/error-handler.js.map +1 -0
- package/dist/cli/gateway.d.ts +14 -0
- package/dist/cli/gateway.d.ts.map +1 -0
- package/dist/cli/gateway.js +222 -0
- package/dist/cli/gateway.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +194 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/action-eval-builder.d.ts +20 -0
- package/dist/core/action-eval-builder.d.ts.map +1 -0
- package/dist/core/action-eval-builder.js +276 -0
- package/dist/core/action-eval-builder.js.map +1 -0
- package/dist/core/agent-fetcher.d.ts +35 -0
- package/dist/core/agent-fetcher.d.ts.map +1 -0
- package/dist/core/agent-fetcher.js +81 -0
- package/dist/core/agent-fetcher.js.map +1 -0
- package/dist/core/api-cache.d.ts +11 -0
- package/dist/core/api-cache.d.ts.map +1 -0
- package/dist/core/api-cache.js +89 -0
- package/dist/core/api-cache.js.map +1 -0
- package/dist/core/config-generator.d.ts +26 -0
- package/dist/core/config-generator.d.ts.map +1 -0
- package/dist/core/config-generator.js +457 -0
- package/dist/core/config-generator.js.map +1 -0
- package/dist/core/conversation-loader.d.ts +21 -0
- package/dist/core/conversation-loader.d.ts.map +1 -0
- package/dist/core/conversation-loader.js +74 -0
- package/dist/core/conversation-loader.js.map +1 -0
- package/dist/core/dataset-loader.d.ts +26 -0
- package/dist/core/dataset-loader.d.ts.map +1 -0
- package/dist/core/dataset-loader.js +121 -0
- package/dist/core/dataset-loader.js.map +1 -0
- package/dist/core/disposition-builder.d.ts +38 -0
- package/dist/core/disposition-builder.d.ts.map +1 -0
- package/dist/core/disposition-builder.js +270 -0
- package/dist/core/disposition-builder.js.map +1 -0
- package/dist/core/eqho-client.d.ts +45 -0
- package/dist/core/eqho-client.d.ts.map +1 -0
- package/dist/core/eqho-client.js +154 -0
- package/dist/core/eqho-client.js.map +1 -0
- package/dist/core/greeting-builder.d.ts +18 -0
- package/dist/core/greeting-builder.d.ts.map +1 -0
- package/dist/core/greeting-builder.js +83 -0
- package/dist/core/greeting-builder.js.map +1 -0
- package/dist/core/postcall-simulator.d.ts +20 -0
- package/dist/core/postcall-simulator.d.ts.map +1 -0
- package/dist/core/postcall-simulator.js +212 -0
- package/dist/core/postcall-simulator.js.map +1 -0
- package/dist/core/prompt-assembler.d.ts +25 -0
- package/dist/core/prompt-assembler.d.ts.map +1 -0
- package/dist/core/prompt-assembler.js +185 -0
- package/dist/core/prompt-assembler.js.map +1 -0
- package/dist/core/promptfoo-runner.d.ts +13 -0
- package/dist/core/promptfoo-runner.d.ts.map +1 -0
- package/dist/core/promptfoo-runner.js +49 -0
- package/dist/core/promptfoo-runner.js.map +1 -0
- package/dist/core/provider-mapper.d.ts +39 -0
- package/dist/core/provider-mapper.d.ts.map +1 -0
- package/dist/core/provider-mapper.js +120 -0
- package/dist/core/provider-mapper.js.map +1 -0
- package/dist/core/template-engine.d.ts +10 -0
- package/dist/core/template-engine.d.ts.map +1 -0
- package/dist/core/template-engine.js +78 -0
- package/dist/core/template-engine.js.map +1 -0
- package/dist/core/tools-builder.d.ts +14 -0
- package/dist/core/tools-builder.d.ts.map +1 -0
- package/dist/core/tools-builder.js +208 -0
- package/dist/core/tools-builder.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/types/config.d.ts +100 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +2 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/eqho.d.ts +221 -0
- package/dist/types/eqho.d.ts.map +1 -0
- package/dist/types/eqho.js +2 -0
- package/dist/types/eqho.js.map +1 -0
- package/dist/types/helpers.d.ts +9 -0
- package/dist/types/helpers.d.ts.map +1 -0
- package/dist/types/helpers.js +8 -0
- package/dist/types/helpers.js.map +1 -0
- package/package.json +77 -0
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
export const diffCommand = new Command("diff")
|
|
6
|
+
.description("Compare two eval result sets and show pass/fail flips and score deltas")
|
|
7
|
+
.argument("<baseline>", "Path to baseline eval-results.json")
|
|
8
|
+
.argument("<candidate>", "Path to candidate eval-results.json")
|
|
9
|
+
.option("--threshold <pct>", "Minimum improvement % to highlight", "0")
|
|
10
|
+
.action(async (baselinePath, candidatePath, opts) => {
|
|
11
|
+
const threshold = parseFloat(opts.threshold) / 100;
|
|
12
|
+
const baseline = loadResults(baselinePath);
|
|
13
|
+
const candidate = loadResults(candidatePath);
|
|
14
|
+
if (!baseline || !candidate) {
|
|
15
|
+
console.log(chalk.red("Failed to load one or both result files."));
|
|
16
|
+
process.exit(1);
|
|
17
|
+
}
|
|
18
|
+
const baseResults = baseline.results?.results ?? [];
|
|
19
|
+
const candResults = candidate.results?.results ?? [];
|
|
20
|
+
const baseStats = baseline.results?.stats ?? baseline.stats;
|
|
21
|
+
const candStats = candidate.results?.stats ?? candidate.stats;
|
|
22
|
+
console.log();
|
|
23
|
+
console.log(chalk.bold("Eval Diff Report"));
|
|
24
|
+
console.log(chalk.dim("─".repeat(60)));
|
|
25
|
+
if (baseStats && candStats) {
|
|
26
|
+
const basePct = baseStats.successes / (baseStats.successes + baseStats.failures + baseStats.errors) * 100;
|
|
27
|
+
const candPct = candStats.successes / (candStats.successes + candStats.failures + candStats.errors) * 100;
|
|
28
|
+
const delta = candPct - basePct;
|
|
29
|
+
const arrow = delta > 0 ? chalk.green("▲") : delta < 0 ? chalk.red("▼") : chalk.dim("=");
|
|
30
|
+
console.log(` Baseline: ${chalk.white(basePct.toFixed(1) + "%")} pass (${baseStats.successes}/${baseStats.successes + baseStats.failures + baseStats.errors})`);
|
|
31
|
+
console.log(` Candidate: ${chalk.white(candPct.toFixed(1) + "%")} pass (${candStats.successes}/${candStats.successes + candStats.failures + candStats.errors})`);
|
|
32
|
+
console.log(` Delta: ${arrow} ${delta > 0 ? "+" : ""}${delta.toFixed(1)}pp`);
|
|
33
|
+
console.log();
|
|
34
|
+
}
|
|
35
|
+
const baseMap = new Map();
|
|
36
|
+
for (const r of baseResults) {
|
|
37
|
+
const key = resultKey(r);
|
|
38
|
+
baseMap.set(key, r);
|
|
39
|
+
}
|
|
40
|
+
const flips = [];
|
|
41
|
+
const scoreDeltas = [];
|
|
42
|
+
for (const cand of candResults) {
|
|
43
|
+
const key = resultKey(cand);
|
|
44
|
+
const base = baseMap.get(key);
|
|
45
|
+
if (!base)
|
|
46
|
+
continue;
|
|
47
|
+
if (base.success !== cand.success) {
|
|
48
|
+
flips.push({
|
|
49
|
+
desc: cand.description || key,
|
|
50
|
+
direction: cand.success ? "improved" : "regressed",
|
|
51
|
+
baseSuccess: base.success,
|
|
52
|
+
candSuccess: cand.success,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
if (base.score != null && cand.score != null && base.score !== cand.score) {
|
|
56
|
+
const delta = cand.score - base.score;
|
|
57
|
+
if (Math.abs(delta) >= threshold) {
|
|
58
|
+
scoreDeltas.push({
|
|
59
|
+
desc: cand.description || key,
|
|
60
|
+
basePct: base.score,
|
|
61
|
+
candPct: cand.score,
|
|
62
|
+
delta,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (flips.length > 0) {
|
|
68
|
+
const improved = flips.filter((f) => f.direction === "improved");
|
|
69
|
+
const regressed = flips.filter((f) => f.direction === "regressed");
|
|
70
|
+
if (improved.length > 0) {
|
|
71
|
+
console.log(chalk.green.bold(`Improved (${improved.length} tests now passing):`));
|
|
72
|
+
for (const f of improved) {
|
|
73
|
+
console.log(chalk.green(` ✓ ${f.desc}`));
|
|
74
|
+
}
|
|
75
|
+
console.log();
|
|
76
|
+
}
|
|
77
|
+
if (regressed.length > 0) {
|
|
78
|
+
console.log(chalk.red.bold(`Regressed (${regressed.length} tests now failing):`));
|
|
79
|
+
for (const f of regressed) {
|
|
80
|
+
console.log(chalk.red(` ✗ ${f.desc}`));
|
|
81
|
+
}
|
|
82
|
+
console.log();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (scoreDeltas.length > 0) {
|
|
86
|
+
console.log(chalk.bold("Score Changes:"));
|
|
87
|
+
const sorted = scoreDeltas.sort((a, b) => b.delta - a.delta);
|
|
88
|
+
for (const s of sorted) {
|
|
89
|
+
const arrow = s.delta > 0 ? chalk.green("▲") : chalk.red("▼");
|
|
90
|
+
console.log(` ${arrow} ${s.desc}: ${(s.basePct * 100).toFixed(0)}% → ${(s.candPct * 100).toFixed(0)}% (${s.delta > 0 ? "+" : ""}${(s.delta * 100).toFixed(1)}pp)`);
|
|
91
|
+
}
|
|
92
|
+
console.log();
|
|
93
|
+
}
|
|
94
|
+
if (flips.length === 0 && scoreDeltas.length === 0) {
|
|
95
|
+
console.log(chalk.dim(" No pass/fail flips or significant score changes."));
|
|
96
|
+
console.log();
|
|
97
|
+
}
|
|
98
|
+
const newTests = candResults.filter((r) => !baseMap.has(resultKey(r)));
|
|
99
|
+
if (newTests.length > 0) {
|
|
100
|
+
console.log(chalk.dim(` ${newTests.length} new test(s) in candidate not found in baseline.`));
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
function resultKey(r) {
|
|
104
|
+
const desc = r.description || "";
|
|
105
|
+
const provider = r.provider?.label || r.provider?.id || "";
|
|
106
|
+
return `${desc}|${provider}`;
|
|
107
|
+
}
|
|
108
|
+
function loadResults(filePath) {
|
|
109
|
+
try {
|
|
110
|
+
const resolved = path.resolve(filePath);
|
|
111
|
+
if (!fs.existsSync(resolved)) {
|
|
112
|
+
console.log(chalk.red(`File not found: ${resolved}`));
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
return JSON.parse(fs.readFileSync(resolved, "utf-8"));
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
console.log(chalk.red(`Failed to parse ${filePath}: ${String(err)}`));
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=diff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"diff.js","sourceRoot":"","sources":["../../../src/cli/commands/diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAoB7B,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;KAC3C,WAAW,CACV,wEAAwE,CACzE;KACA,QAAQ,CAAC,YAAY,EAAE,oCAAoC,CAAC;KAC5D,QAAQ,CAAC,aAAa,EAAE,qCAAqC,CAAC;KAC9D,MAAM,CAAC,mBAAmB,EAAE,oCAAoC,EAAE,GAAG,CAAC;KACtE,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,aAAqB,EAAE,IAAI,EAAE,EAAE;IAClE,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;IAEnD,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;IAE7C,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IACpD,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IAErD,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;IAC5D,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC;IAE9D,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEvC,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC1G,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC1G,MAAM,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzF,OAAO,CAAC,GAAG,CACT,eAAe,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CACpJ,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gBAAgB,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CACrJ,CAAC;QACF,OAAO,CAAC,GAAG,CACT,YAAY,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACjE,CAAC;QACF,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,KAAK,GAKN,EAAE,CAAC;IAER,MAAM,WAAW,GAKZ,EAAE,CAAC;IAER,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;gBAC7B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;gBAClD,WAAW,EAAE,IAAI,CAAC,OAAO;gBACzB,WAAW,EAAE,IAAI,CAAC,OAAO;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;oBAC7B,OAAO,EAAE,IAAI,CAAC,KAAK;oBACnB,OAAO,EAAE,IAAI,CAAC,KAAK;oBACnB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC;QAEnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,sBAAsB,CAAC,CACrE,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,MAAM,sBAAsB,CAAC,CACrE,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CACvJ,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC;IACjG,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS,SAAS,CAAC,CAAa;IAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;IAC3D,OAAO,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB;IACnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAe,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
interface CheckResult {
|
|
3
|
+
name: string;
|
|
4
|
+
ok: boolean;
|
|
5
|
+
detail: string;
|
|
6
|
+
fix?: string;
|
|
7
|
+
}
|
|
8
|
+
declare function runChecks(): Promise<CheckResult[]>;
|
|
9
|
+
export declare const doctorCommand: Command;
|
|
10
|
+
export { runChecks };
|
|
11
|
+
//# sourceMappingURL=doctor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,iBAAe,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CA0MjD;AA+DD,eAAO,MAAM,aAAa,SAwDtB,CAAC;AAEL,OAAO,EAAE,SAAS,EAAE,CAAC"}
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
6
|
+
import { loadAuth } from "../auth-store.js";
|
|
7
|
+
import { EqhoClient } from "../../core/eqho-client.js";
|
|
8
|
+
import { resolvePromptfoo, getPromptfooVersion } from "../../core/promptfoo-runner.js";
|
|
9
|
+
async function runChecks() {
|
|
10
|
+
const results = [];
|
|
11
|
+
// 1. Node.js version
|
|
12
|
+
const nodeVersion = process.version;
|
|
13
|
+
const major = parseInt(nodeVersion.slice(1).split(".")[0], 10);
|
|
14
|
+
results.push({
|
|
15
|
+
name: "Node.js",
|
|
16
|
+
ok: major >= 20,
|
|
17
|
+
detail: `${nodeVersion} ${major >= 20 ? "(>=20 required)" : ""}`,
|
|
18
|
+
fix: major < 20 ? "Upgrade Node.js to v20+: https://nodejs.org" : undefined,
|
|
19
|
+
});
|
|
20
|
+
// 2. eqho-eval version
|
|
21
|
+
let selfVersion = "unknown";
|
|
22
|
+
try {
|
|
23
|
+
const pkgPath = path.resolve(import.meta.dirname ?? __dirname, "..", "..", "..", "package.json");
|
|
24
|
+
if (fs.existsSync(pkgPath)) {
|
|
25
|
+
const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
|
|
26
|
+
selfVersion = pkg.version ?? "unknown";
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch { /* ignore */ }
|
|
30
|
+
let latestVersion = null;
|
|
31
|
+
try {
|
|
32
|
+
const out = execFileSync("npm", ["view", "eqho-eval", "version"], {
|
|
33
|
+
stdio: "pipe",
|
|
34
|
+
encoding: "utf-8",
|
|
35
|
+
timeout: 5000,
|
|
36
|
+
});
|
|
37
|
+
latestVersion = out.trim();
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
// npm registry lookup failed — skip
|
|
41
|
+
}
|
|
42
|
+
const versionDetail = latestVersion && latestVersion !== selfVersion
|
|
43
|
+
? `v${selfVersion} (latest: v${latestVersion})`
|
|
44
|
+
: `v${selfVersion}`;
|
|
45
|
+
results.push({
|
|
46
|
+
name: "eqho-eval",
|
|
47
|
+
ok: true,
|
|
48
|
+
detail: versionDetail,
|
|
49
|
+
fix: latestVersion && latestVersion !== selfVersion
|
|
50
|
+
? `npm install -g eqho-eval@${latestVersion}`
|
|
51
|
+
: undefined,
|
|
52
|
+
});
|
|
53
|
+
// 3. Eqho auth configured
|
|
54
|
+
const auth = loadAuth();
|
|
55
|
+
const authDetail = auth
|
|
56
|
+
? auth.authMode === "bearer"
|
|
57
|
+
? `SSO (${auth.email ?? "bearer"})`
|
|
58
|
+
: `API key (${(auth.apiKey ?? "").slice(0, 8)}...)`
|
|
59
|
+
: "not configured";
|
|
60
|
+
results.push({
|
|
61
|
+
name: "Eqho auth",
|
|
62
|
+
ok: !!auth,
|
|
63
|
+
detail: authDetail,
|
|
64
|
+
fix: !auth ? "eqho-eval auth --key <api-key> or eqho-eval auth --login" : undefined,
|
|
65
|
+
});
|
|
66
|
+
// 4. Eqho API reachable
|
|
67
|
+
if (auth) {
|
|
68
|
+
const client = new EqhoClient(auth);
|
|
69
|
+
let reachable = false;
|
|
70
|
+
let orgDetail = "";
|
|
71
|
+
try {
|
|
72
|
+
const { campaigns } = await client.listCampaigns(1);
|
|
73
|
+
reachable = true;
|
|
74
|
+
orgDetail = campaigns.length > 0 ? ` (${campaigns.length}+ campaigns)` : "";
|
|
75
|
+
}
|
|
76
|
+
catch { /* ignore */ }
|
|
77
|
+
results.push({
|
|
78
|
+
name: "Eqho API",
|
|
79
|
+
ok: reachable,
|
|
80
|
+
detail: reachable ? `reachable${orgDetail}` : "unreachable",
|
|
81
|
+
fix: !reachable ? "Check your API key and network connection" : undefined,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
results.push({
|
|
86
|
+
name: "Eqho API",
|
|
87
|
+
ok: false,
|
|
88
|
+
detail: "skipped (no API key)",
|
|
89
|
+
fix: "eqho-eval auth --key <your-api-key>",
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
// 5. Backend proxy
|
|
93
|
+
if (auth?.backendUrl && auth?.backendToken) {
|
|
94
|
+
let backendOk = false;
|
|
95
|
+
try {
|
|
96
|
+
if (auth.apiKey) {
|
|
97
|
+
const res = await fetch(`${auth.backendUrl}/api/auth/validate`, {
|
|
98
|
+
method: "POST",
|
|
99
|
+
headers: { "Content-Type": "application/json" },
|
|
100
|
+
body: JSON.stringify({ apiKey: auth.apiKey }),
|
|
101
|
+
signal: AbortSignal.timeout(5000),
|
|
102
|
+
});
|
|
103
|
+
backendOk = res.ok;
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
const res = await fetch(`${auth.backendUrl}/api/eqho/v1/campaigns?limit=1`, {
|
|
107
|
+
headers: { Authorization: `Bearer ${auth.backendToken}` },
|
|
108
|
+
signal: AbortSignal.timeout(5000),
|
|
109
|
+
});
|
|
110
|
+
backendOk = res.ok;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
catch { /* ignore */ }
|
|
114
|
+
results.push({
|
|
115
|
+
name: "Backend proxy",
|
|
116
|
+
ok: backendOk,
|
|
117
|
+
detail: backendOk
|
|
118
|
+
? `connected (${auth.backendUrl})`
|
|
119
|
+
: `unreachable (${auth.backendUrl})`,
|
|
120
|
+
fix: !backendOk ? "eqho-eval auth --key <key> --backend <url>" : undefined,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
results.push({
|
|
125
|
+
name: "Backend proxy",
|
|
126
|
+
ok: false,
|
|
127
|
+
detail: "not configured (direct mode — requires OPENAI_API_KEY)",
|
|
128
|
+
fix: "eqho-eval auth --key <key> --backend https://evals.eqho-solutions.dev",
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
// 5b. Org context
|
|
132
|
+
if (auth?.orgId) {
|
|
133
|
+
results.push({
|
|
134
|
+
name: "Org context",
|
|
135
|
+
ok: true,
|
|
136
|
+
detail: auth.orgName ? `${auth.orgName} (${auth.orgId})` : auth.orgId,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
// 6. promptfoo installed
|
|
140
|
+
const pfoo = resolvePromptfoo();
|
|
141
|
+
const pfooVersion = pfoo ? getPromptfooVersion() : null;
|
|
142
|
+
let pfooOk = !!pfoo;
|
|
143
|
+
let pfooDetail = pfoo
|
|
144
|
+
? `installed via ${pfoo.source}${pfooVersion ? ` (v${pfooVersion})` : ""}`
|
|
145
|
+
: "not found";
|
|
146
|
+
let pfooFix = !pfoo ? "npm install promptfoo" : undefined;
|
|
147
|
+
// 6b. Check better-sqlite3 native module (common blocker)
|
|
148
|
+
if (pfoo) {
|
|
149
|
+
const sqliteOk = checkBetterSqlite3();
|
|
150
|
+
if (!sqliteOk) {
|
|
151
|
+
pfooOk = false;
|
|
152
|
+
pfooDetail += " (better-sqlite3 native module broken)";
|
|
153
|
+
pfooFix = "eqho-eval doctor --fix or npm rebuild better-sqlite3";
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
results.push({
|
|
157
|
+
name: "promptfoo",
|
|
158
|
+
ok: pfooOk,
|
|
159
|
+
detail: pfooDetail,
|
|
160
|
+
fix: pfooFix,
|
|
161
|
+
});
|
|
162
|
+
// 6. OpenAI API key set
|
|
163
|
+
const hasOpenAI = !!process.env.OPENAI_API_KEY;
|
|
164
|
+
const dotEnvPath = path.join(process.cwd(), ".env");
|
|
165
|
+
let dotEnvHasKey = false;
|
|
166
|
+
if (!hasOpenAI && fs.existsSync(dotEnvPath)) {
|
|
167
|
+
try {
|
|
168
|
+
const content = fs.readFileSync(dotEnvPath, "utf-8");
|
|
169
|
+
dotEnvHasKey = content.includes("OPENAI_API_KEY=");
|
|
170
|
+
}
|
|
171
|
+
catch { /* ignore */ }
|
|
172
|
+
}
|
|
173
|
+
results.push({
|
|
174
|
+
name: "OpenAI API key",
|
|
175
|
+
ok: hasOpenAI || dotEnvHasKey,
|
|
176
|
+
detail: hasOpenAI
|
|
177
|
+
? "set in environment"
|
|
178
|
+
: dotEnvHasKey
|
|
179
|
+
? "found in .env"
|
|
180
|
+
: "not set",
|
|
181
|
+
fix: !(hasOpenAI || dotEnvHasKey)
|
|
182
|
+
? "export OPENAI_API_KEY=sk-... or add to .env"
|
|
183
|
+
: undefined,
|
|
184
|
+
});
|
|
185
|
+
// 7. Project config
|
|
186
|
+
const configPath = path.join(process.cwd(), "eqho.config.json");
|
|
187
|
+
const pfooConfigPath = path.join(process.cwd(), "promptfooconfig.yaml");
|
|
188
|
+
const hasProjectConfig = fs.existsSync(configPath);
|
|
189
|
+
const hasPfooConfig = fs.existsSync(pfooConfigPath);
|
|
190
|
+
results.push({
|
|
191
|
+
name: "Project config",
|
|
192
|
+
ok: hasProjectConfig || hasPfooConfig,
|
|
193
|
+
detail: hasProjectConfig
|
|
194
|
+
? "eqho.config.json found"
|
|
195
|
+
: hasPfooConfig
|
|
196
|
+
? "promptfooconfig.yaml found (no eqho.config.json)"
|
|
197
|
+
: "no config found in current directory",
|
|
198
|
+
fix: !(hasProjectConfig || hasPfooConfig)
|
|
199
|
+
? "eqho-eval init --campaign <id>"
|
|
200
|
+
: undefined,
|
|
201
|
+
});
|
|
202
|
+
return results;
|
|
203
|
+
}
|
|
204
|
+
function checkBetterSqlite3() {
|
|
205
|
+
try {
|
|
206
|
+
execFileSync("node", ["-e", "require('better-sqlite3')"], {
|
|
207
|
+
stdio: "pipe",
|
|
208
|
+
timeout: 5000,
|
|
209
|
+
});
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async function fixBetterSqlite3() {
|
|
217
|
+
console.log(chalk.dim(" Attempting to rebuild better-sqlite3..."));
|
|
218
|
+
const rebuild = spawnSync("npm", ["rebuild", "better-sqlite3"], {
|
|
219
|
+
stdio: "inherit",
|
|
220
|
+
timeout: 60_000,
|
|
221
|
+
});
|
|
222
|
+
if (rebuild.status === 0 && checkBetterSqlite3()) {
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
console.log(chalk.dim(" Rebuild failed. Attempting prebuild-install..."));
|
|
226
|
+
const prebuild = spawnSync("npx", ["prebuild-install", "--runtime", "napi", "--target", "6"], {
|
|
227
|
+
stdio: "inherit",
|
|
228
|
+
timeout: 60_000,
|
|
229
|
+
cwd: findBetterSqlitePath() ?? process.cwd(),
|
|
230
|
+
});
|
|
231
|
+
if (prebuild.status === 0 && checkBetterSqlite3()) {
|
|
232
|
+
return true;
|
|
233
|
+
}
|
|
234
|
+
console.log(chalk.dim(" Auto-fix failed. Manual steps:"));
|
|
235
|
+
console.log(chalk.dim(" 1. xcode-select --install (macOS only)"));
|
|
236
|
+
console.log(chalk.dim(" 2. npm rebuild better-sqlite3"));
|
|
237
|
+
console.log(chalk.dim(" 3. Or reinstall promptfoo: npm i -g promptfoo"));
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
function findBetterSqlitePath() {
|
|
241
|
+
const candidates = [
|
|
242
|
+
path.join(process.cwd(), "node_modules", "better-sqlite3"),
|
|
243
|
+
path.resolve(import.meta.dirname ?? __dirname, "..", "..", "..", "node_modules", "better-sqlite3"),
|
|
244
|
+
];
|
|
245
|
+
try {
|
|
246
|
+
const globalRoot = execFileSync("npm", ["root", "-g"], { stdio: "pipe", encoding: "utf-8" }).trim();
|
|
247
|
+
candidates.push(path.join(globalRoot, "promptfoo", "node_modules", "better-sqlite3"));
|
|
248
|
+
candidates.push(path.join(globalRoot, "better-sqlite3"));
|
|
249
|
+
}
|
|
250
|
+
catch { /* ignore */ }
|
|
251
|
+
for (const p of candidates) {
|
|
252
|
+
if (fs.existsSync(p))
|
|
253
|
+
return p;
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
export const doctorCommand = new Command("doctor")
|
|
258
|
+
.description("Check your environment and diagnose issues")
|
|
259
|
+
.option("--fix", "Attempt to auto-repair common issues")
|
|
260
|
+
.action(async (opts, cmd) => {
|
|
261
|
+
const jsonMode = cmd.parent?.opts().json;
|
|
262
|
+
const fixMode = opts.fix;
|
|
263
|
+
const results = await runChecks();
|
|
264
|
+
if (jsonMode && !fixMode) {
|
|
265
|
+
console.log(JSON.stringify(results, null, 2));
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
console.log();
|
|
269
|
+
let passed = results.filter((r) => r.ok).length;
|
|
270
|
+
for (const r of results) {
|
|
271
|
+
const icon = r.ok ? chalk.green("✓") : chalk.red("✗");
|
|
272
|
+
console.log(` ${icon} ${chalk.bold(r.name)} ${chalk.dim(r.detail)}`);
|
|
273
|
+
if (r.fix) {
|
|
274
|
+
console.log(chalk.cyan(` → ${r.fix}`));
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
if (fixMode) {
|
|
278
|
+
console.log();
|
|
279
|
+
console.log(chalk.bold(" Running auto-fix..."));
|
|
280
|
+
let fixed = 0;
|
|
281
|
+
// Fix better-sqlite3
|
|
282
|
+
const sqliteResult = results.find((r) => r.name === "promptfoo" && r.detail.includes("better-sqlite3"));
|
|
283
|
+
if (sqliteResult && !sqliteResult.ok) {
|
|
284
|
+
const ok = await fixBetterSqlite3();
|
|
285
|
+
if (ok) {
|
|
286
|
+
console.log(chalk.green(" ✓ better-sqlite3 fixed"));
|
|
287
|
+
fixed++;
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
console.log(chalk.red(" ✗ better-sqlite3 could not be auto-fixed"));
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (fixed > 0) {
|
|
294
|
+
console.log(chalk.green(`\n Fixed ${fixed} issue(s). Run 'eqho-eval doctor' to verify.`));
|
|
295
|
+
passed += fixed;
|
|
296
|
+
}
|
|
297
|
+
else if (!sqliteResult) {
|
|
298
|
+
console.log(chalk.dim(" No auto-fixable issues found."));
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
console.log();
|
|
302
|
+
console.log(passed === results.length
|
|
303
|
+
? chalk.green(` ${passed}/${results.length} checks passed`)
|
|
304
|
+
: chalk.yellow(` ${passed}/${results.length} checks passed`));
|
|
305
|
+
console.log();
|
|
306
|
+
});
|
|
307
|
+
export { runChecks };
|
|
308
|
+
//# sourceMappingURL=doctor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AASvF,KAAK,UAAU,SAAS;IACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,qBAAqB;IACrB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,SAAS;QACf,EAAE,EAAE,KAAK,IAAI,EAAE;QACf,MAAM,EAAE,GAAG,WAAW,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE;QAChE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,SAAS;KAC5E,CAAC,CAAC;IAEH,uBAAuB;IACvB,IAAI,WAAW,GAAG,SAAS,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACjG,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1D,WAAW,GAAG,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC;QACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE;YAChE,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;IACtC,CAAC;IAED,MAAM,aAAa,GAAG,aAAa,IAAI,aAAa,KAAK,WAAW;QAClE,CAAC,CAAC,IAAI,WAAW,cAAc,aAAa,GAAG;QAC/C,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC;IAEtB,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,IAAI;QACR,MAAM,EAAE,aAAa;QACrB,GAAG,EAAE,aAAa,IAAI,aAAa,KAAK,WAAW;YACjD,CAAC,CAAC,4BAA4B,aAAa,EAAE;YAC7C,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,0BAA0B;IAC1B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,IAAI;QACrB,CAAC,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAC1B,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,IAAI,QAAQ,GAAG;YACnC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM;QACrD,CAAC,CAAC,gBAAgB,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,CAAC,CAAC,IAAI;QACV,MAAM,EAAE,UAAU;QAClB,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,0DAA0D,CAAC,CAAC,CAAC,SAAS;KACpF,CAAC,CAAC;IAEH,wBAAwB;IACxB,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACpD,SAAS,GAAG,IAAI,CAAC;YACjB,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,EAAE,EAAE,SAAS;YACb,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,SAAS,EAAE,CAAC,CAAC,CAAC,aAAa;YAC3D,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,SAAS;SAC1E,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,sBAAsB;YAC9B,GAAG,EAAE,qCAAqC;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,IAAI,IAAI,EAAE,UAAU,IAAI,IAAI,EAAE,YAAY,EAAE,CAAC;QAC3C,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,oBAAoB,EAAE;oBAC9D,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC7C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;iBAClC,CAAC,CAAC;gBACH,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,gCAAgC,EAAE;oBAC1E,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,YAAY,EAAE,EAAE;oBACzD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;iBAClC,CAAC,CAAC;gBACH,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,eAAe;YACrB,EAAE,EAAE,SAAS;YACb,MAAM,EAAE,SAAS;gBACf,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,GAAG;gBAClC,CAAC,CAAC,gBAAgB,IAAI,CAAC,UAAU,GAAG;YACtC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC,SAAS;SAC3E,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,eAAe;YACrB,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,wDAAwD;YAChE,GAAG,EAAE,uEAAuE;SAC7E,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAI,IAAI,EAAE,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;SACtE,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC;IACpB,IAAI,UAAU,GAAG,IAAI;QACnB,CAAC,CAAC,iBAAiB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1E,CAAC,CAAC,WAAW,CAAC;IAChB,IAAI,OAAO,GAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9E,0DAA0D;IAC1D,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAC;QACtC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,GAAG,KAAK,CAAC;YACf,UAAU,IAAI,wCAAwC,CAAC;YACvD,OAAO,GAAG,wDAAwD,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,UAAU;QAClB,GAAG,EAAE,OAAO;KACb,CAAC,CAAC;IAEH,wBAAwB;IACxB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,gBAAgB;QACtB,EAAE,EAAE,SAAS,IAAI,YAAY;QAC7B,MAAM,EAAE,SAAS;YACf,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,YAAY;gBACZ,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,SAAS;QACf,GAAG,EAAE,CAAC,CAAC,SAAS,IAAI,YAAY,CAAC;YAC/B,CAAC,CAAC,6CAA6C;YAC/C,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,oBAAoB;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,gBAAgB;QACtB,EAAE,EAAE,gBAAgB,IAAI,aAAa;QACrC,MAAM,EAAE,gBAAgB;YACtB,CAAC,CAAC,wBAAwB;YAC1B,CAAC,CAAC,aAAa;gBACb,CAAC,CAAC,kDAAkD;gBACpD,CAAC,CAAC,sCAAsC;QAC5C,GAAG,EAAE,CAAC,CAAC,gBAAgB,IAAI,aAAa,CAAC;YACvC,CAAC,CAAC,gCAAgC;YAClC,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB;IACzB,IAAI,CAAC;QACH,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,2BAA2B,CAAC,EAAE;YACxD,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE;QAC9D,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,MAAM;KAChB,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,EAAE,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,SAAS,CACxB,KAAK,EACL,CAAC,kBAAkB,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,EAC1D;QACE,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,MAAM;QACf,GAAG,EAAE,oBAAoB,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE;KAC7C,CACF,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,EAAE,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB;IAC3B,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,gBAAgB,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,gBAAgB,CAAC;KACnG,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACtF,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;KAC/C,WAAW,CAAC,4CAA4C,CAAC;KACzD,MAAM,CAAC,OAAO,EAAE,sCAAsC,CAAC;KACvD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC;IACzB,MAAM,OAAO,GAAG,MAAM,SAAS,EAAE,CAAC;IAElC,IAAI,QAAQ,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAEhD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACjD,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,qBAAqB;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACxG,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;YACpC,IAAI,EAAE,EAAE,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;gBACrD,KAAK,EAAE,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,KAAK,8CAA8C,CAAC,CAAC,CAAC;YAC3F,MAAM,IAAI,KAAK,CAAC;QAClB,CAAC;aAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CACT,MAAM,KAAK,OAAO,CAAC,MAAM;QACvB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC;QAC5D,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC,CAChE,CAAC;IACF,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC,CAAC,CAAC;AAEL,OAAO,EAAE,SAAS,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,eAAO,MAAM,WAAW,SAyFpB,CAAC"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { runPromptfoo, isPromptfooInstalled } from "../../core/promptfoo-runner.js";
|
|
6
|
+
export const evalCommand = new Command("eval")
|
|
7
|
+
.description("Run promptfoo evaluations")
|
|
8
|
+
.option("-c, --config <path>", "Path to promptfooconfig.yaml")
|
|
9
|
+
.option("-o, --output <file>", "Output file (json, yaml, csv, html)")
|
|
10
|
+
.option("-w, --watch", "Watch mode — re-run on file changes")
|
|
11
|
+
.option("--no-cache", "Disable promptfoo cache")
|
|
12
|
+
.option("--no-progress-bar", "Disable progress bar")
|
|
13
|
+
.option("--share", "Upload results to promptfoo cloud after eval")
|
|
14
|
+
.option("--filter-failing", "Only re-run previously failing tests")
|
|
15
|
+
.option("--repeat <n>", "Repeat each test N times")
|
|
16
|
+
.option("--delay <ms>", "Delay between API calls in ms")
|
|
17
|
+
.option("--verbose", "Enable verbose output")
|
|
18
|
+
.allowUnknownOption(true)
|
|
19
|
+
.action(async (opts, cmd) => {
|
|
20
|
+
const jsonMode = cmd.parent?.opts().json;
|
|
21
|
+
if (!isPromptfooInstalled()) {
|
|
22
|
+
if (jsonMode) {
|
|
23
|
+
console.log(JSON.stringify({ error: "promptfoo is not installed", fix: "npm install promptfoo" }));
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
console.log(chalk.red("promptfoo is not installed."));
|
|
27
|
+
console.log();
|
|
28
|
+
console.log(chalk.dim(" Install it:"));
|
|
29
|
+
console.log(chalk.cyan(" npm install promptfoo"));
|
|
30
|
+
console.log();
|
|
31
|
+
}
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
const configPath = opts.config || "promptfooconfig.yaml";
|
|
35
|
+
if (!fs.existsSync(path.resolve(configPath))) {
|
|
36
|
+
if (jsonMode) {
|
|
37
|
+
console.log(JSON.stringify({ error: `Config not found: ${configPath}`, fix: "eqho-eval init --campaign <id>" }));
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
console.log(chalk.red(`Config not found: ${configPath}`));
|
|
41
|
+
console.log();
|
|
42
|
+
console.log(chalk.dim(" Generate one:"));
|
|
43
|
+
console.log(chalk.cyan(" eqho-eval init --campaign <id>"));
|
|
44
|
+
console.log();
|
|
45
|
+
}
|
|
46
|
+
process.exit(1);
|
|
47
|
+
}
|
|
48
|
+
const args = ["eval"];
|
|
49
|
+
if (opts.config)
|
|
50
|
+
args.push("-c", opts.config);
|
|
51
|
+
if (opts.output)
|
|
52
|
+
args.push("-o", opts.output);
|
|
53
|
+
if (opts.watch)
|
|
54
|
+
args.push("--watch");
|
|
55
|
+
if (opts.cache === false)
|
|
56
|
+
args.push("--no-cache");
|
|
57
|
+
if (opts.progressBar === false)
|
|
58
|
+
args.push("--no-progress-bar");
|
|
59
|
+
if (opts.share)
|
|
60
|
+
args.push("--share");
|
|
61
|
+
if (opts.filterFailing)
|
|
62
|
+
args.push("--filter-failing");
|
|
63
|
+
if (opts.repeat)
|
|
64
|
+
args.push("--repeat", opts.repeat);
|
|
65
|
+
if (opts.delay)
|
|
66
|
+
args.push("--delay", opts.delay);
|
|
67
|
+
if (opts.verbose)
|
|
68
|
+
args.push("--verbose");
|
|
69
|
+
const jsonOutputPath = opts.output?.endsWith(".json") ? opts.output : null;
|
|
70
|
+
if (!jsonOutputPath && !opts.output) {
|
|
71
|
+
args.push("-o", "output/eval-results.json");
|
|
72
|
+
}
|
|
73
|
+
const extraArgs = cmd.args || [];
|
|
74
|
+
args.push(...extraArgs);
|
|
75
|
+
const { exitCode } = runPromptfoo(args);
|
|
76
|
+
if (jsonMode) {
|
|
77
|
+
const resultsPath = jsonOutputPath || "output/eval-results.json";
|
|
78
|
+
try {
|
|
79
|
+
const results = JSON.parse(fs.readFileSync(path.resolve(resultsPath), "utf-8"));
|
|
80
|
+
const stats = results.results?.stats || results.stats;
|
|
81
|
+
if (stats) {
|
|
82
|
+
const summary = {
|
|
83
|
+
pass: stats.successes ?? 0,
|
|
84
|
+
fail: stats.failures ?? 0,
|
|
85
|
+
errors: stats.errors ?? 0,
|
|
86
|
+
total: (stats.successes ?? 0) + (stats.failures ?? 0) + (stats.errors ?? 0),
|
|
87
|
+
passRate: stats.successes && (stats.successes + stats.failures) > 0
|
|
88
|
+
? +(stats.successes / (stats.successes + stats.failures)).toFixed(4)
|
|
89
|
+
: 0,
|
|
90
|
+
exitCode,
|
|
91
|
+
};
|
|
92
|
+
console.log(JSON.stringify(summary));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
console.log(JSON.stringify({ exitCode, error: "Could not read eval results" }));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
process.exit(exitCode);
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AAEpF,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;KAC3C,WAAW,CAAC,2BAA2B,CAAC;KACxC,MAAM,CAAC,qBAAqB,EAAE,8BAA8B,CAAC;KAC7D,MAAM,CAAC,qBAAqB,EAAE,qCAAqC,CAAC;KACpE,MAAM,CAAC,aAAa,EAAE,qCAAqC,CAAC;KAC5D,MAAM,CAAC,YAAY,EAAE,yBAAyB,CAAC;KAC/C,MAAM,CAAC,mBAAmB,EAAE,sBAAsB,CAAC;KACnD,MAAM,CAAC,SAAS,EAAE,8CAA8C,CAAC;KACjE,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC;KAClE,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,cAAc,EAAE,+BAA+B,CAAC;KACvD,MAAM,CAAC,WAAW,EAAE,uBAAuB,CAAC;KAC5C,kBAAkB,CAAC,IAAI,CAAC;KACxB,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IAEzC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC5B,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,4BAA4B,EAAE,GAAG,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,IAAI,sBAAsB,CAAC;IACzD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,qBAAqB,UAAU,EAAE,EAAE,GAAG,EAAE,gCAAgC,EAAE,CAAC,CAAC,CAAC;QACnH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClD,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,aAAa;QAAE,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IACjC,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,WAAW,GAAG,cAAc,IAAI,0BAA0B,CAAC;QACjE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAChF,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;YACtD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,OAAO,GAAG;oBACd,IAAI,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;oBACzB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;oBACzB,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;oBAC3E,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;wBACjE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;wBACpE,CAAC,CAAC,CAAC;oBACL,QAAQ;iBACT,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,6BAA6B,EAAE,CAAC,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACzB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,eAAO,MAAM,WAAW,SA4LpB,CAAC"}
|