eqho-eval 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +552 -0
  3. package/dist/cli/auth-store.d.ts +5 -0
  4. package/dist/cli/auth-store.d.ts.map +1 -0
  5. package/dist/cli/auth-store.js +39 -0
  6. package/dist/cli/auth-store.js.map +1 -0
  7. package/dist/cli/banner.d.ts +3 -0
  8. package/dist/cli/banner.d.ts.map +1 -0
  9. package/dist/cli/banner.js +38 -0
  10. package/dist/cli/banner.js.map +1 -0
  11. package/dist/cli/commands/action-eval.d.ts +3 -0
  12. package/dist/cli/commands/action-eval.d.ts.map +1 -0
  13. package/dist/cli/commands/action-eval.js +133 -0
  14. package/dist/cli/commands/action-eval.js.map +1 -0
  15. package/dist/cli/commands/auth.d.ts +3 -0
  16. package/dist/cli/commands/auth.d.ts.map +1 -0
  17. package/dist/cli/commands/auth.js +156 -0
  18. package/dist/cli/commands/auth.js.map +1 -0
  19. package/dist/cli/commands/cache.d.ts +3 -0
  20. package/dist/cli/commands/cache.d.ts.map +1 -0
  21. package/dist/cli/commands/cache.js +43 -0
  22. package/dist/cli/commands/cache.js.map +1 -0
  23. package/dist/cli/commands/ci.d.ts +3 -0
  24. package/dist/cli/commands/ci.d.ts.map +1 -0
  25. package/dist/cli/commands/ci.js +124 -0
  26. package/dist/cli/commands/ci.js.map +1 -0
  27. package/dist/cli/commands/conversations.d.ts +3 -0
  28. package/dist/cli/commands/conversations.d.ts.map +1 -0
  29. package/dist/cli/commands/conversations.js +89 -0
  30. package/dist/cli/commands/conversations.js.map +1 -0
  31. package/dist/cli/commands/diff.d.ts +3 -0
  32. package/dist/cli/commands/diff.d.ts.map +1 -0
  33. package/dist/cli/commands/diff.js +122 -0
  34. package/dist/cli/commands/diff.js.map +1 -0
  35. package/dist/cli/commands/doctor.d.ts +11 -0
  36. package/dist/cli/commands/doctor.d.ts.map +1 -0
  37. package/dist/cli/commands/doctor.js +308 -0
  38. package/dist/cli/commands/doctor.js.map +1 -0
  39. package/dist/cli/commands/eval.d.ts +3 -0
  40. package/dist/cli/commands/eval.d.ts.map +1 -0
  41. package/dist/cli/commands/eval.js +101 -0
  42. package/dist/cli/commands/eval.js.map +1 -0
  43. package/dist/cli/commands/init.d.ts +3 -0
  44. package/dist/cli/commands/init.d.ts.map +1 -0
  45. package/dist/cli/commands/init.js +182 -0
  46. package/dist/cli/commands/init.js.map +1 -0
  47. package/dist/cli/commands/list.d.ts +3 -0
  48. package/dist/cli/commands/list.d.ts.map +1 -0
  49. package/dist/cli/commands/list.js +80 -0
  50. package/dist/cli/commands/list.js.map +1 -0
  51. package/dist/cli/commands/mentions.d.ts +3 -0
  52. package/dist/cli/commands/mentions.d.ts.map +1 -0
  53. package/dist/cli/commands/mentions.js +125 -0
  54. package/dist/cli/commands/mentions.js.map +1 -0
  55. package/dist/cli/commands/org.d.ts +3 -0
  56. package/dist/cli/commands/org.d.ts.map +1 -0
  57. package/dist/cli/commands/org.js +196 -0
  58. package/dist/cli/commands/org.js.map +1 -0
  59. package/dist/cli/commands/postcall-eval.d.ts +3 -0
  60. package/dist/cli/commands/postcall-eval.d.ts.map +1 -0
  61. package/dist/cli/commands/postcall-eval.js +188 -0
  62. package/dist/cli/commands/postcall-eval.js.map +1 -0
  63. package/dist/cli/commands/render.d.ts +3 -0
  64. package/dist/cli/commands/render.d.ts.map +1 -0
  65. package/dist/cli/commands/render.js +223 -0
  66. package/dist/cli/commands/render.js.map +1 -0
  67. package/dist/cli/commands/results.d.ts +3 -0
  68. package/dist/cli/commands/results.d.ts.map +1 -0
  69. package/dist/cli/commands/results.js +128 -0
  70. package/dist/cli/commands/results.js.map +1 -0
  71. package/dist/cli/commands/scenarios.d.ts +3 -0
  72. package/dist/cli/commands/scenarios.d.ts.map +1 -0
  73. package/dist/cli/commands/scenarios.js +57 -0
  74. package/dist/cli/commands/scenarios.js.map +1 -0
  75. package/dist/cli/commands/start.d.ts +3 -0
  76. package/dist/cli/commands/start.d.ts.map +1 -0
  77. package/dist/cli/commands/start.js +260 -0
  78. package/dist/cli/commands/start.js.map +1 -0
  79. package/dist/cli/commands/status.d.ts +3 -0
  80. package/dist/cli/commands/status.d.ts.map +1 -0
  81. package/dist/cli/commands/status.js +133 -0
  82. package/dist/cli/commands/status.js.map +1 -0
  83. package/dist/cli/commands/sync.d.ts +3 -0
  84. package/dist/cli/commands/sync.d.ts.map +1 -0
  85. package/dist/cli/commands/sync.js +80 -0
  86. package/dist/cli/commands/sync.js.map +1 -0
  87. package/dist/cli/commands/view.d.ts +3 -0
  88. package/dist/cli/commands/view.d.ts.map +1 -0
  89. package/dist/cli/commands/view.js +29 -0
  90. package/dist/cli/commands/view.js.map +1 -0
  91. package/dist/cli/error-handler.d.ts +8 -0
  92. package/dist/cli/error-handler.d.ts.map +1 -0
  93. package/dist/cli/error-handler.js +133 -0
  94. package/dist/cli/error-handler.js.map +1 -0
  95. package/dist/cli/gateway.d.ts +14 -0
  96. package/dist/cli/gateway.d.ts.map +1 -0
  97. package/dist/cli/gateway.js +222 -0
  98. package/dist/cli/gateway.js.map +1 -0
  99. package/dist/cli/index.d.ts +3 -0
  100. package/dist/cli/index.d.ts.map +1 -0
  101. package/dist/cli/index.js +194 -0
  102. package/dist/cli/index.js.map +1 -0
  103. package/dist/core/action-eval-builder.d.ts +20 -0
  104. package/dist/core/action-eval-builder.d.ts.map +1 -0
  105. package/dist/core/action-eval-builder.js +276 -0
  106. package/dist/core/action-eval-builder.js.map +1 -0
  107. package/dist/core/agent-fetcher.d.ts +35 -0
  108. package/dist/core/agent-fetcher.d.ts.map +1 -0
  109. package/dist/core/agent-fetcher.js +81 -0
  110. package/dist/core/agent-fetcher.js.map +1 -0
  111. package/dist/core/api-cache.d.ts +11 -0
  112. package/dist/core/api-cache.d.ts.map +1 -0
  113. package/dist/core/api-cache.js +89 -0
  114. package/dist/core/api-cache.js.map +1 -0
  115. package/dist/core/config-generator.d.ts +26 -0
  116. package/dist/core/config-generator.d.ts.map +1 -0
  117. package/dist/core/config-generator.js +457 -0
  118. package/dist/core/config-generator.js.map +1 -0
  119. package/dist/core/conversation-loader.d.ts +21 -0
  120. package/dist/core/conversation-loader.d.ts.map +1 -0
  121. package/dist/core/conversation-loader.js +74 -0
  122. package/dist/core/conversation-loader.js.map +1 -0
  123. package/dist/core/dataset-loader.d.ts +26 -0
  124. package/dist/core/dataset-loader.d.ts.map +1 -0
  125. package/dist/core/dataset-loader.js +121 -0
  126. package/dist/core/dataset-loader.js.map +1 -0
  127. package/dist/core/disposition-builder.d.ts +38 -0
  128. package/dist/core/disposition-builder.d.ts.map +1 -0
  129. package/dist/core/disposition-builder.js +270 -0
  130. package/dist/core/disposition-builder.js.map +1 -0
  131. package/dist/core/eqho-client.d.ts +45 -0
  132. package/dist/core/eqho-client.d.ts.map +1 -0
  133. package/dist/core/eqho-client.js +154 -0
  134. package/dist/core/eqho-client.js.map +1 -0
  135. package/dist/core/greeting-builder.d.ts +18 -0
  136. package/dist/core/greeting-builder.d.ts.map +1 -0
  137. package/dist/core/greeting-builder.js +83 -0
  138. package/dist/core/greeting-builder.js.map +1 -0
  139. package/dist/core/postcall-simulator.d.ts +20 -0
  140. package/dist/core/postcall-simulator.d.ts.map +1 -0
  141. package/dist/core/postcall-simulator.js +212 -0
  142. package/dist/core/postcall-simulator.js.map +1 -0
  143. package/dist/core/prompt-assembler.d.ts +25 -0
  144. package/dist/core/prompt-assembler.d.ts.map +1 -0
  145. package/dist/core/prompt-assembler.js +185 -0
  146. package/dist/core/prompt-assembler.js.map +1 -0
  147. package/dist/core/promptfoo-runner.d.ts +13 -0
  148. package/dist/core/promptfoo-runner.d.ts.map +1 -0
  149. package/dist/core/promptfoo-runner.js +49 -0
  150. package/dist/core/promptfoo-runner.js.map +1 -0
  151. package/dist/core/provider-mapper.d.ts +39 -0
  152. package/dist/core/provider-mapper.d.ts.map +1 -0
  153. package/dist/core/provider-mapper.js +120 -0
  154. package/dist/core/provider-mapper.js.map +1 -0
  155. package/dist/core/template-engine.d.ts +10 -0
  156. package/dist/core/template-engine.d.ts.map +1 -0
  157. package/dist/core/template-engine.js +78 -0
  158. package/dist/core/template-engine.js.map +1 -0
  159. package/dist/core/tools-builder.d.ts +14 -0
  160. package/dist/core/tools-builder.d.ts.map +1 -0
  161. package/dist/core/tools-builder.js +208 -0
  162. package/dist/core/tools-builder.js.map +1 -0
  163. package/dist/index.d.ts +18 -0
  164. package/dist/index.d.ts.map +1 -0
  165. package/dist/index.js +16 -0
  166. package/dist/index.js.map +1 -0
  167. package/dist/types/config.d.ts +100 -0
  168. package/dist/types/config.d.ts.map +1 -0
  169. package/dist/types/config.js +2 -0
  170. package/dist/types/config.js.map +1 -0
  171. package/dist/types/eqho.d.ts +221 -0
  172. package/dist/types/eqho.d.ts.map +1 -0
  173. package/dist/types/eqho.js +2 -0
  174. package/dist/types/eqho.js.map +1 -0
  175. package/dist/types/helpers.d.ts +9 -0
  176. package/dist/types/helpers.d.ts.map +1 -0
  177. package/dist/types/helpers.js +8 -0
  178. package/dist/types/helpers.js.map +1 -0
  179. package/package.json +77 -0
@@ -0,0 +1,122 @@
1
+ import { Command } from "commander";
2
+ import chalk from "chalk";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ export const diffCommand = new Command("diff")
6
+ .description("Compare two eval result sets and show pass/fail flips and score deltas")
7
+ .argument("<baseline>", "Path to baseline eval-results.json")
8
+ .argument("<candidate>", "Path to candidate eval-results.json")
9
+ .option("--threshold <pct>", "Minimum improvement % to highlight", "0")
10
+ .action(async (baselinePath, candidatePath, opts) => {
11
+ const threshold = parseFloat(opts.threshold) / 100;
12
+ const baseline = loadResults(baselinePath);
13
+ const candidate = loadResults(candidatePath);
14
+ if (!baseline || !candidate) {
15
+ console.log(chalk.red("Failed to load one or both result files."));
16
+ process.exit(1);
17
+ }
18
+ const baseResults = baseline.results?.results ?? [];
19
+ const candResults = candidate.results?.results ?? [];
20
+ const baseStats = baseline.results?.stats ?? baseline.stats;
21
+ const candStats = candidate.results?.stats ?? candidate.stats;
22
+ console.log();
23
+ console.log(chalk.bold("Eval Diff Report"));
24
+ console.log(chalk.dim("─".repeat(60)));
25
+ if (baseStats && candStats) {
26
+ const basePct = baseStats.successes / (baseStats.successes + baseStats.failures + baseStats.errors) * 100;
27
+ const candPct = candStats.successes / (candStats.successes + candStats.failures + candStats.errors) * 100;
28
+ const delta = candPct - basePct;
29
+ const arrow = delta > 0 ? chalk.green("▲") : delta < 0 ? chalk.red("▼") : chalk.dim("=");
30
+ console.log(` Baseline: ${chalk.white(basePct.toFixed(1) + "%")} pass (${baseStats.successes}/${baseStats.successes + baseStats.failures + baseStats.errors})`);
31
+ console.log(` Candidate: ${chalk.white(candPct.toFixed(1) + "%")} pass (${candStats.successes}/${candStats.successes + candStats.failures + candStats.errors})`);
32
+ console.log(` Delta: ${arrow} ${delta > 0 ? "+" : ""}${delta.toFixed(1)}pp`);
33
+ console.log();
34
+ }
35
+ const baseMap = new Map();
36
+ for (const r of baseResults) {
37
+ const key = resultKey(r);
38
+ baseMap.set(key, r);
39
+ }
40
+ const flips = [];
41
+ const scoreDeltas = [];
42
+ for (const cand of candResults) {
43
+ const key = resultKey(cand);
44
+ const base = baseMap.get(key);
45
+ if (!base)
46
+ continue;
47
+ if (base.success !== cand.success) {
48
+ flips.push({
49
+ desc: cand.description || key,
50
+ direction: cand.success ? "improved" : "regressed",
51
+ baseSuccess: base.success,
52
+ candSuccess: cand.success,
53
+ });
54
+ }
55
+ if (base.score != null && cand.score != null && base.score !== cand.score) {
56
+ const delta = cand.score - base.score;
57
+ if (Math.abs(delta) >= threshold) {
58
+ scoreDeltas.push({
59
+ desc: cand.description || key,
60
+ basePct: base.score,
61
+ candPct: cand.score,
62
+ delta,
63
+ });
64
+ }
65
+ }
66
+ }
67
+ if (flips.length > 0) {
68
+ const improved = flips.filter((f) => f.direction === "improved");
69
+ const regressed = flips.filter((f) => f.direction === "regressed");
70
+ if (improved.length > 0) {
71
+ console.log(chalk.green.bold(`Improved (${improved.length} tests now passing):`));
72
+ for (const f of improved) {
73
+ console.log(chalk.green(` ✓ ${f.desc}`));
74
+ }
75
+ console.log();
76
+ }
77
+ if (regressed.length > 0) {
78
+ console.log(chalk.red.bold(`Regressed (${regressed.length} tests now failing):`));
79
+ for (const f of regressed) {
80
+ console.log(chalk.red(` ✗ ${f.desc}`));
81
+ }
82
+ console.log();
83
+ }
84
+ }
85
+ if (scoreDeltas.length > 0) {
86
+ console.log(chalk.bold("Score Changes:"));
87
+ const sorted = scoreDeltas.sort((a, b) => b.delta - a.delta);
88
+ for (const s of sorted) {
89
+ const arrow = s.delta > 0 ? chalk.green("▲") : chalk.red("▼");
90
+ console.log(` ${arrow} ${s.desc}: ${(s.basePct * 100).toFixed(0)}% → ${(s.candPct * 100).toFixed(0)}% (${s.delta > 0 ? "+" : ""}${(s.delta * 100).toFixed(1)}pp)`);
91
+ }
92
+ console.log();
93
+ }
94
+ if (flips.length === 0 && scoreDeltas.length === 0) {
95
+ console.log(chalk.dim(" No pass/fail flips or significant score changes."));
96
+ console.log();
97
+ }
98
+ const newTests = candResults.filter((r) => !baseMap.has(resultKey(r)));
99
+ if (newTests.length > 0) {
100
+ console.log(chalk.dim(` ${newTests.length} new test(s) in candidate not found in baseline.`));
101
+ }
102
+ });
103
+ function resultKey(r) {
104
+ const desc = r.description || "";
105
+ const provider = r.provider?.label || r.provider?.id || "";
106
+ return `${desc}|${provider}`;
107
+ }
108
+ function loadResults(filePath) {
109
+ try {
110
+ const resolved = path.resolve(filePath);
111
+ if (!fs.existsSync(resolved)) {
112
+ console.log(chalk.red(`File not found: ${resolved}`));
113
+ return null;
114
+ }
115
+ return JSON.parse(fs.readFileSync(resolved, "utf-8"));
116
+ }
117
+ catch (err) {
118
+ console.log(chalk.red(`Failed to parse ${filePath}: ${String(err)}`));
119
+ return null;
120
+ }
121
+ }
122
+ //# sourceMappingURL=diff.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"diff.js","sourceRoot":"","sources":["../../../src/cli/commands/diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAoB7B,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;KAC3C,WAAW,CACV,wEAAwE,CACzE;KACA,QAAQ,CAAC,YAAY,EAAE,oCAAoC,CAAC;KAC5D,QAAQ,CAAC,aAAa,EAAE,qCAAqC,CAAC;KAC9D,MAAM,CAAC,mBAAmB,EAAE,oCAAoC,EAAE,GAAG,CAAC;KACtE,MAAM,CAAC,KAAK,EAAE,YAAoB,EAAE,aAAqB,EAAE,IAAI,EAAE,EAAE;IAClE,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;IAEnD,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;IAE7C,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IACpD,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;IAErD,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;IAC5D,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC;IAE9D,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEvC,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC1G,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,GAAG,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC1G,MAAM,KAAK,GAAG,OAAO,GAAG,OAAO,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzF,OAAO,CAAC,GAAG,CACT,eAAe,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CACpJ,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gBAAgB,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,CACrJ,CAAC;QACF,OAAO,CAAC,GAAG,CACT,YAAY,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACjE,CAAC;QACF,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,KAAK,GAKN,EAAE,CAAC;IAER,MAAM,WAAW,GAKZ,EAAE,CAAC;IAER,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;gBAC7B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;gBAClD,WAAW,EAAE,IAAI,CAAC,OAAO;gBACzB,WAAW,EAAE,IAAI,CAAC,OAAO;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;YACtC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjC,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,IAAI,CAAC,WAAW,IAAI,GAAG;oBAC7B,OAAO,EAAE,IAAI,CAAC,KAAK;oBACnB,OAAO,EAAE,IAAI,CAAC,KAAK;oBACnB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC;QAEnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,sBAAsB,CAAC,CACrE,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,MAAM,sBAAsB,CAAC,CACrE,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CACvJ,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,MAAM,kDAAkD,CAAC,CAAC,CAAC;IACjG,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS,SAAS,CAAC,CAAa;IAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC;IAC3D,OAAO,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB;IACnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC,CAAC;YACtD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAe,CAAC;IACtE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mBAAmB,QAAQ,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
@@ -0,0 +1,11 @@
1
+ import { Command } from "commander";
2
+ interface CheckResult {
3
+ name: string;
4
+ ok: boolean;
5
+ detail: string;
6
+ fix?: string;
7
+ }
8
+ declare function runChecks(): Promise<CheckResult[]>;
9
+ export declare const doctorCommand: Command;
10
+ export { runChecks };
11
+ //# sourceMappingURL=doctor.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,iBAAe,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CA0MjD;AA+DD,eAAO,MAAM,aAAa,SAwDtB,CAAC;AAEL,OAAO,EAAE,SAAS,EAAE,CAAC"}
@@ -0,0 +1,308 @@
1
+ import { Command } from "commander";
2
+ import chalk from "chalk";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ import { execFileSync, spawnSync } from "node:child_process";
6
+ import { loadAuth } from "../auth-store.js";
7
+ import { EqhoClient } from "../../core/eqho-client.js";
8
+ import { resolvePromptfoo, getPromptfooVersion } from "../../core/promptfoo-runner.js";
9
+ async function runChecks() {
10
+ const results = [];
11
+ // 1. Node.js version
12
+ const nodeVersion = process.version;
13
+ const major = parseInt(nodeVersion.slice(1).split(".")[0], 10);
14
+ results.push({
15
+ name: "Node.js",
16
+ ok: major >= 20,
17
+ detail: `${nodeVersion} ${major >= 20 ? "(>=20 required)" : ""}`,
18
+ fix: major < 20 ? "Upgrade Node.js to v20+: https://nodejs.org" : undefined,
19
+ });
20
+ // 2. eqho-eval version
21
+ let selfVersion = "unknown";
22
+ try {
23
+ const pkgPath = path.resolve(import.meta.dirname ?? __dirname, "..", "..", "..", "package.json");
24
+ if (fs.existsSync(pkgPath)) {
25
+ const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
26
+ selfVersion = pkg.version ?? "unknown";
27
+ }
28
+ }
29
+ catch { /* ignore */ }
30
+ let latestVersion = null;
31
+ try {
32
+ const out = execFileSync("npm", ["view", "eqho-eval", "version"], {
33
+ stdio: "pipe",
34
+ encoding: "utf-8",
35
+ timeout: 5000,
36
+ });
37
+ latestVersion = out.trim();
38
+ }
39
+ catch {
40
+ // npm registry lookup failed — skip
41
+ }
42
+ const versionDetail = latestVersion && latestVersion !== selfVersion
43
+ ? `v${selfVersion} (latest: v${latestVersion})`
44
+ : `v${selfVersion}`;
45
+ results.push({
46
+ name: "eqho-eval",
47
+ ok: true,
48
+ detail: versionDetail,
49
+ fix: latestVersion && latestVersion !== selfVersion
50
+ ? `npm install -g eqho-eval@${latestVersion}`
51
+ : undefined,
52
+ });
53
+ // 3. Eqho auth configured
54
+ const auth = loadAuth();
55
+ const authDetail = auth
56
+ ? auth.authMode === "bearer"
57
+ ? `SSO (${auth.email ?? "bearer"})`
58
+ : `API key (${(auth.apiKey ?? "").slice(0, 8)}...)`
59
+ : "not configured";
60
+ results.push({
61
+ name: "Eqho auth",
62
+ ok: !!auth,
63
+ detail: authDetail,
64
+ fix: !auth ? "eqho-eval auth --key <api-key> or eqho-eval auth --login" : undefined,
65
+ });
66
+ // 4. Eqho API reachable
67
+ if (auth) {
68
+ const client = new EqhoClient(auth);
69
+ let reachable = false;
70
+ let orgDetail = "";
71
+ try {
72
+ const { campaigns } = await client.listCampaigns(1);
73
+ reachable = true;
74
+ orgDetail = campaigns.length > 0 ? ` (${campaigns.length}+ campaigns)` : "";
75
+ }
76
+ catch { /* ignore */ }
77
+ results.push({
78
+ name: "Eqho API",
79
+ ok: reachable,
80
+ detail: reachable ? `reachable${orgDetail}` : "unreachable",
81
+ fix: !reachable ? "Check your API key and network connection" : undefined,
82
+ });
83
+ }
84
+ else {
85
+ results.push({
86
+ name: "Eqho API",
87
+ ok: false,
88
+ detail: "skipped (no API key)",
89
+ fix: "eqho-eval auth --key <your-api-key>",
90
+ });
91
+ }
92
+ // 5. Backend proxy
93
+ if (auth?.backendUrl && auth?.backendToken) {
94
+ let backendOk = false;
95
+ try {
96
+ if (auth.apiKey) {
97
+ const res = await fetch(`${auth.backendUrl}/api/auth/validate`, {
98
+ method: "POST",
99
+ headers: { "Content-Type": "application/json" },
100
+ body: JSON.stringify({ apiKey: auth.apiKey }),
101
+ signal: AbortSignal.timeout(5000),
102
+ });
103
+ backendOk = res.ok;
104
+ }
105
+ else {
106
+ const res = await fetch(`${auth.backendUrl}/api/eqho/v1/campaigns?limit=1`, {
107
+ headers: { Authorization: `Bearer ${auth.backendToken}` },
108
+ signal: AbortSignal.timeout(5000),
109
+ });
110
+ backendOk = res.ok;
111
+ }
112
+ }
113
+ catch { /* ignore */ }
114
+ results.push({
115
+ name: "Backend proxy",
116
+ ok: backendOk,
117
+ detail: backendOk
118
+ ? `connected (${auth.backendUrl})`
119
+ : `unreachable (${auth.backendUrl})`,
120
+ fix: !backendOk ? "eqho-eval auth --key <key> --backend <url>" : undefined,
121
+ });
122
+ }
123
+ else {
124
+ results.push({
125
+ name: "Backend proxy",
126
+ ok: false,
127
+ detail: "not configured (direct mode — requires OPENAI_API_KEY)",
128
+ fix: "eqho-eval auth --key <key> --backend https://evals.eqho-solutions.dev",
129
+ });
130
+ }
131
+ // 5b. Org context
132
+ if (auth?.orgId) {
133
+ results.push({
134
+ name: "Org context",
135
+ ok: true,
136
+ detail: auth.orgName ? `${auth.orgName} (${auth.orgId})` : auth.orgId,
137
+ });
138
+ }
139
+ // 6. promptfoo installed
140
+ const pfoo = resolvePromptfoo();
141
+ const pfooVersion = pfoo ? getPromptfooVersion() : null;
142
+ let pfooOk = !!pfoo;
143
+ let pfooDetail = pfoo
144
+ ? `installed via ${pfoo.source}${pfooVersion ? ` (v${pfooVersion})` : ""}`
145
+ : "not found";
146
+ let pfooFix = !pfoo ? "npm install promptfoo" : undefined;
147
+ // 6b. Check better-sqlite3 native module (common blocker)
148
+ if (pfoo) {
149
+ const sqliteOk = checkBetterSqlite3();
150
+ if (!sqliteOk) {
151
+ pfooOk = false;
152
+ pfooDetail += " (better-sqlite3 native module broken)";
153
+ pfooFix = "eqho-eval doctor --fix or npm rebuild better-sqlite3";
154
+ }
155
+ }
156
+ results.push({
157
+ name: "promptfoo",
158
+ ok: pfooOk,
159
+ detail: pfooDetail,
160
+ fix: pfooFix,
161
+ });
162
+ // 6. OpenAI API key set
163
+ const hasOpenAI = !!process.env.OPENAI_API_KEY;
164
+ const dotEnvPath = path.join(process.cwd(), ".env");
165
+ let dotEnvHasKey = false;
166
+ if (!hasOpenAI && fs.existsSync(dotEnvPath)) {
167
+ try {
168
+ const content = fs.readFileSync(dotEnvPath, "utf-8");
169
+ dotEnvHasKey = content.includes("OPENAI_API_KEY=");
170
+ }
171
+ catch { /* ignore */ }
172
+ }
173
+ results.push({
174
+ name: "OpenAI API key",
175
+ ok: hasOpenAI || dotEnvHasKey,
176
+ detail: hasOpenAI
177
+ ? "set in environment"
178
+ : dotEnvHasKey
179
+ ? "found in .env"
180
+ : "not set",
181
+ fix: !(hasOpenAI || dotEnvHasKey)
182
+ ? "export OPENAI_API_KEY=sk-... or add to .env"
183
+ : undefined,
184
+ });
185
+ // 7. Project config
186
+ const configPath = path.join(process.cwd(), "eqho.config.json");
187
+ const pfooConfigPath = path.join(process.cwd(), "promptfooconfig.yaml");
188
+ const hasProjectConfig = fs.existsSync(configPath);
189
+ const hasPfooConfig = fs.existsSync(pfooConfigPath);
190
+ results.push({
191
+ name: "Project config",
192
+ ok: hasProjectConfig || hasPfooConfig,
193
+ detail: hasProjectConfig
194
+ ? "eqho.config.json found"
195
+ : hasPfooConfig
196
+ ? "promptfooconfig.yaml found (no eqho.config.json)"
197
+ : "no config found in current directory",
198
+ fix: !(hasProjectConfig || hasPfooConfig)
199
+ ? "eqho-eval init --campaign <id>"
200
+ : undefined,
201
+ });
202
+ return results;
203
+ }
204
+ function checkBetterSqlite3() {
205
+ try {
206
+ execFileSync("node", ["-e", "require('better-sqlite3')"], {
207
+ stdio: "pipe",
208
+ timeout: 5000,
209
+ });
210
+ return true;
211
+ }
212
+ catch {
213
+ return false;
214
+ }
215
+ }
216
+ async function fixBetterSqlite3() {
217
+ console.log(chalk.dim(" Attempting to rebuild better-sqlite3..."));
218
+ const rebuild = spawnSync("npm", ["rebuild", "better-sqlite3"], {
219
+ stdio: "inherit",
220
+ timeout: 60_000,
221
+ });
222
+ if (rebuild.status === 0 && checkBetterSqlite3()) {
223
+ return true;
224
+ }
225
+ console.log(chalk.dim(" Rebuild failed. Attempting prebuild-install..."));
226
+ const prebuild = spawnSync("npx", ["prebuild-install", "--runtime", "napi", "--target", "6"], {
227
+ stdio: "inherit",
228
+ timeout: 60_000,
229
+ cwd: findBetterSqlitePath() ?? process.cwd(),
230
+ });
231
+ if (prebuild.status === 0 && checkBetterSqlite3()) {
232
+ return true;
233
+ }
234
+ console.log(chalk.dim(" Auto-fix failed. Manual steps:"));
235
+ console.log(chalk.dim(" 1. xcode-select --install (macOS only)"));
236
+ console.log(chalk.dim(" 2. npm rebuild better-sqlite3"));
237
+ console.log(chalk.dim(" 3. Or reinstall promptfoo: npm i -g promptfoo"));
238
+ return false;
239
+ }
240
+ function findBetterSqlitePath() {
241
+ const candidates = [
242
+ path.join(process.cwd(), "node_modules", "better-sqlite3"),
243
+ path.resolve(import.meta.dirname ?? __dirname, "..", "..", "..", "node_modules", "better-sqlite3"),
244
+ ];
245
+ try {
246
+ const globalRoot = execFileSync("npm", ["root", "-g"], { stdio: "pipe", encoding: "utf-8" }).trim();
247
+ candidates.push(path.join(globalRoot, "promptfoo", "node_modules", "better-sqlite3"));
248
+ candidates.push(path.join(globalRoot, "better-sqlite3"));
249
+ }
250
+ catch { /* ignore */ }
251
+ for (const p of candidates) {
252
+ if (fs.existsSync(p))
253
+ return p;
254
+ }
255
+ return null;
256
+ }
257
+ export const doctorCommand = new Command("doctor")
258
+ .description("Check your environment and diagnose issues")
259
+ .option("--fix", "Attempt to auto-repair common issues")
260
+ .action(async (opts, cmd) => {
261
+ const jsonMode = cmd.parent?.opts().json;
262
+ const fixMode = opts.fix;
263
+ const results = await runChecks();
264
+ if (jsonMode && !fixMode) {
265
+ console.log(JSON.stringify(results, null, 2));
266
+ return;
267
+ }
268
+ console.log();
269
+ let passed = results.filter((r) => r.ok).length;
270
+ for (const r of results) {
271
+ const icon = r.ok ? chalk.green("✓") : chalk.red("✗");
272
+ console.log(` ${icon} ${chalk.bold(r.name)} ${chalk.dim(r.detail)}`);
273
+ if (r.fix) {
274
+ console.log(chalk.cyan(` → ${r.fix}`));
275
+ }
276
+ }
277
+ if (fixMode) {
278
+ console.log();
279
+ console.log(chalk.bold(" Running auto-fix..."));
280
+ let fixed = 0;
281
+ // Fix better-sqlite3
282
+ const sqliteResult = results.find((r) => r.name === "promptfoo" && r.detail.includes("better-sqlite3"));
283
+ if (sqliteResult && !sqliteResult.ok) {
284
+ const ok = await fixBetterSqlite3();
285
+ if (ok) {
286
+ console.log(chalk.green(" ✓ better-sqlite3 fixed"));
287
+ fixed++;
288
+ }
289
+ else {
290
+ console.log(chalk.red(" ✗ better-sqlite3 could not be auto-fixed"));
291
+ }
292
+ }
293
+ if (fixed > 0) {
294
+ console.log(chalk.green(`\n Fixed ${fixed} issue(s). Run 'eqho-eval doctor' to verify.`));
295
+ passed += fixed;
296
+ }
297
+ else if (!sqliteResult) {
298
+ console.log(chalk.dim(" No auto-fixable issues found."));
299
+ }
300
+ }
301
+ console.log();
302
+ console.log(passed === results.length
303
+ ? chalk.green(` ${passed}/${results.length} checks passed`)
304
+ : chalk.yellow(` ${passed}/${results.length} checks passed`));
305
+ console.log();
306
+ });
307
+ export { runChecks };
308
+ //# sourceMappingURL=doctor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AASvF,KAAK,UAAU,SAAS;IACtB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,qBAAqB;IACrB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,SAAS;QACf,EAAE,EAAE,KAAK,IAAI,EAAE;QACf,MAAM,EAAE,GAAG,WAAW,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE;QAChE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,SAAS;KAC5E,CAAC,CAAC;IAEH,uBAAuB;IACvB,IAAI,WAAW,GAAG,SAAS,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QACjG,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC1D,WAAW,GAAG,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC;QACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE;YAChE,KAAK,EAAE,MAAM;YACb,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;IACtC,CAAC;IAED,MAAM,aAAa,GAAG,aAAa,IAAI,aAAa,KAAK,WAAW;QAClE,CAAC,CAAC,IAAI,WAAW,cAAc,aAAa,GAAG;QAC/C,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC;IAEtB,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,IAAI;QACR,MAAM,EAAE,aAAa;QACrB,GAAG,EAAE,aAAa,IAAI,aAAa,KAAK,WAAW;YACjD,CAAC,CAAC,4BAA4B,aAAa,EAAE;YAC7C,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,0BAA0B;IAC1B,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,MAAM,UAAU,GAAG,IAAI;QACrB,CAAC,CAAC,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAC1B,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,IAAI,QAAQ,GAAG;YACnC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM;QACrD,CAAC,CAAC,gBAAgB,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,CAAC,CAAC,IAAI;QACV,MAAM,EAAE,UAAU;QAClB,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,0DAA0D,CAAC,CAAC,CAAC,SAAS;KACpF,CAAC,CAAC;IAEH,wBAAwB;IACxB,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACpD,SAAS,GAAG,IAAI,CAAC;YACjB,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,EAAE,EAAE,SAAS;YACb,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,SAAS,EAAE,CAAC,CAAC,CAAC,aAAa;YAC3D,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC,CAAC,SAAS;SAC1E,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,UAAU;YAChB,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,sBAAsB;YAC9B,GAAG,EAAE,qCAAqC;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,IAAI,IAAI,EAAE,UAAU,IAAI,IAAI,EAAE,YAAY,EAAE,CAAC;QAC3C,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,oBAAoB,EAAE;oBAC9D,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC7C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;iBAClC,CAAC,CAAC;gBACH,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,gCAAgC,EAAE;oBAC1E,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,YAAY,EAAE,EAAE;oBACzD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;iBAClC,CAAC,CAAC;gBACH,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,eAAe;YACrB,EAAE,EAAE,SAAS;YACb,MAAM,EAAE,SAAS;gBACf,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,GAAG;gBAClC,CAAC,CAAC,gBAAgB,IAAI,CAAC,UAAU,GAAG;YACtC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC,SAAS;SAC3E,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,eAAe;YACrB,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,wDAAwD;YAChE,GAAG,EAAE,uEAAuE;SAC7E,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAI,IAAI,EAAE,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;SACtE,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,MAAM,WAAW,GAAG,IAAI,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxD,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC;IACpB,IAAI,UAAU,GAAG,IAAI;QACnB,CAAC,CAAC,iBAAiB,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1E,CAAC,CAAC,WAAW,CAAC;IAChB,IAAI,OAAO,GAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,SAAS,CAAC;IAE9E,0DAA0D;IAC1D,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,QAAQ,GAAG,kBAAkB,EAAE,CAAC;QACtC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,GAAG,KAAK,CAAC;YACf,UAAU,IAAI,wCAAwC,CAAC;YACvD,OAAO,GAAG,wDAAwD,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,WAAW;QACjB,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,UAAU;QAClB,GAAG,EAAE,OAAO;KACb,CAAC,CAAC;IAEH,wBAAwB;IACxB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,gBAAgB;QACtB,EAAE,EAAE,SAAS,IAAI,YAAY;QAC7B,MAAM,EAAE,SAAS;YACf,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,YAAY;gBACZ,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,SAAS;QACf,GAAG,EAAE,CAAC,CAAC,SAAS,IAAI,YAAY,CAAC;YAC/B,CAAC,CAAC,6CAA6C;YAC/C,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,oBAAoB;IACpB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,kBAAkB,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAC;IACxE,MAAM,gBAAgB,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACnD,MAAM,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACpD,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,gBAAgB;QACtB,EAAE,EAAE,gBAAgB,IAAI,aAAa;QACrC,MAAM,EAAE,gBAAgB;YACtB,CAAC,CAAC,wBAAwB;YAC1B,CAAC,CAAC,aAAa;gBACb,CAAC,CAAC,kDAAkD;gBACpD,CAAC,CAAC,sCAAsC;QAC5C,GAAG,EAAE,CAAC,CAAC,gBAAgB,IAAI,aAAa,CAAC;YACvC,CAAC,CAAC,gCAAgC;YAClC,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB;IACzB,IAAI,CAAC;QACH,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,2BAA2B,CAAC,EAAE;YACxD,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,gBAAgB,CAAC,EAAE;QAC9D,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,MAAM;KAChB,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,EAAE,EAAE,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,SAAS,CACxB,KAAK,EACL,CAAC,kBAAkB,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,EAC1D;QACE,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,MAAM;QACf,GAAG,EAAE,oBAAoB,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE;KAC7C,CACF,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB,EAAE,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,oBAAoB;IAC3B,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,gBAAgB,CAAC;QAC1D,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,gBAAgB,CAAC;KACnG,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACtF,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAExB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;KAC/C,WAAW,CAAC,4CAA4C,CAAC;KACzD,MAAM,CAAC,OAAO,EAAE,sCAAsC,CAAC;KACvD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC;IACzB,MAAM,OAAO,GAAG,MAAM,SAAS,EAAE,CAAC;IAElC,IAAI,QAAQ,IAAI,CAAC,OAAO,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,IAAI,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IAEhD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;YACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACjD,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,qBAAqB;QACrB,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACxG,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;YACpC,IAAI,EAAE,EAAE,CAAC;gBACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;gBACrD,KAAK,EAAE,CAAC;YACV,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,KAAK,8CAA8C,CAAC,CAAC,CAAC;YAC3F,MAAM,IAAI,KAAK,CAAC;QAClB,CAAC;aAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CACT,MAAM,KAAK,OAAO,CAAC,MAAM;QACvB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC;QAC5D,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,MAAM,gBAAgB,CAAC,CAChE,CAAC;IACF,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC,CAAC,CAAC;AAEL,OAAO,EAAE,SAAS,EAAE,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare const evalCommand: Command;
3
+ //# sourceMappingURL=eval.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,eAAO,MAAM,WAAW,SAyFpB,CAAC"}
@@ -0,0 +1,101 @@
1
+ import { Command } from "commander";
2
+ import chalk from "chalk";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ import { runPromptfoo, isPromptfooInstalled } from "../../core/promptfoo-runner.js";
6
+ export const evalCommand = new Command("eval")
7
+ .description("Run promptfoo evaluations")
8
+ .option("-c, --config <path>", "Path to promptfooconfig.yaml")
9
+ .option("-o, --output <file>", "Output file (json, yaml, csv, html)")
10
+ .option("-w, --watch", "Watch mode — re-run on file changes")
11
+ .option("--no-cache", "Disable promptfoo cache")
12
+ .option("--no-progress-bar", "Disable progress bar")
13
+ .option("--share", "Upload results to promptfoo cloud after eval")
14
+ .option("--filter-failing", "Only re-run previously failing tests")
15
+ .option("--repeat <n>", "Repeat each test N times")
16
+ .option("--delay <ms>", "Delay between API calls in ms")
17
+ .option("--verbose", "Enable verbose output")
18
+ .allowUnknownOption(true)
19
+ .action(async (opts, cmd) => {
20
+ const jsonMode = cmd.parent?.opts().json;
21
+ if (!isPromptfooInstalled()) {
22
+ if (jsonMode) {
23
+ console.log(JSON.stringify({ error: "promptfoo is not installed", fix: "npm install promptfoo" }));
24
+ }
25
+ else {
26
+ console.log(chalk.red("promptfoo is not installed."));
27
+ console.log();
28
+ console.log(chalk.dim(" Install it:"));
29
+ console.log(chalk.cyan(" npm install promptfoo"));
30
+ console.log();
31
+ }
32
+ process.exit(1);
33
+ }
34
+ const configPath = opts.config || "promptfooconfig.yaml";
35
+ if (!fs.existsSync(path.resolve(configPath))) {
36
+ if (jsonMode) {
37
+ console.log(JSON.stringify({ error: `Config not found: ${configPath}`, fix: "eqho-eval init --campaign <id>" }));
38
+ }
39
+ else {
40
+ console.log(chalk.red(`Config not found: ${configPath}`));
41
+ console.log();
42
+ console.log(chalk.dim(" Generate one:"));
43
+ console.log(chalk.cyan(" eqho-eval init --campaign <id>"));
44
+ console.log();
45
+ }
46
+ process.exit(1);
47
+ }
48
+ const args = ["eval"];
49
+ if (opts.config)
50
+ args.push("-c", opts.config);
51
+ if (opts.output)
52
+ args.push("-o", opts.output);
53
+ if (opts.watch)
54
+ args.push("--watch");
55
+ if (opts.cache === false)
56
+ args.push("--no-cache");
57
+ if (opts.progressBar === false)
58
+ args.push("--no-progress-bar");
59
+ if (opts.share)
60
+ args.push("--share");
61
+ if (opts.filterFailing)
62
+ args.push("--filter-failing");
63
+ if (opts.repeat)
64
+ args.push("--repeat", opts.repeat);
65
+ if (opts.delay)
66
+ args.push("--delay", opts.delay);
67
+ if (opts.verbose)
68
+ args.push("--verbose");
69
+ const jsonOutputPath = opts.output?.endsWith(".json") ? opts.output : null;
70
+ if (!jsonOutputPath && !opts.output) {
71
+ args.push("-o", "output/eval-results.json");
72
+ }
73
+ const extraArgs = cmd.args || [];
74
+ args.push(...extraArgs);
75
+ const { exitCode } = runPromptfoo(args);
76
+ if (jsonMode) {
77
+ const resultsPath = jsonOutputPath || "output/eval-results.json";
78
+ try {
79
+ const results = JSON.parse(fs.readFileSync(path.resolve(resultsPath), "utf-8"));
80
+ const stats = results.results?.stats || results.stats;
81
+ if (stats) {
82
+ const summary = {
83
+ pass: stats.successes ?? 0,
84
+ fail: stats.failures ?? 0,
85
+ errors: stats.errors ?? 0,
86
+ total: (stats.successes ?? 0) + (stats.failures ?? 0) + (stats.errors ?? 0),
87
+ passRate: stats.successes && (stats.successes + stats.failures) > 0
88
+ ? +(stats.successes / (stats.successes + stats.failures)).toFixed(4)
89
+ : 0,
90
+ exitCode,
91
+ };
92
+ console.log(JSON.stringify(summary));
93
+ }
94
+ }
95
+ catch {
96
+ console.log(JSON.stringify({ exitCode, error: "Could not read eval results" }));
97
+ }
98
+ }
99
+ process.exit(exitCode);
100
+ });
101
+ //# sourceMappingURL=eval.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,gCAAgC,CAAC;AAEpF,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC;KAC3C,WAAW,CAAC,2BAA2B,CAAC;KACxC,MAAM,CAAC,qBAAqB,EAAE,8BAA8B,CAAC;KAC7D,MAAM,CAAC,qBAAqB,EAAE,qCAAqC,CAAC;KACpE,MAAM,CAAC,aAAa,EAAE,qCAAqC,CAAC;KAC5D,MAAM,CAAC,YAAY,EAAE,yBAAyB,CAAC;KAC/C,MAAM,CAAC,mBAAmB,EAAE,sBAAsB,CAAC;KACnD,MAAM,CAAC,SAAS,EAAE,8CAA8C,CAAC;KACjE,MAAM,CAAC,kBAAkB,EAAE,sCAAsC,CAAC;KAClE,MAAM,CAAC,cAAc,EAAE,0BAA0B,CAAC;KAClD,MAAM,CAAC,cAAc,EAAE,+BAA+B,CAAC;KACvD,MAAM,CAAC,WAAW,EAAE,uBAAuB,CAAC;KAC5C,kBAAkB,CAAC,IAAI,CAAC;KACxB,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IAEzC,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC5B,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,4BAA4B,EAAE,GAAG,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC;QACrG,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,IAAI,sBAAsB,CAAC;IACzD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC7C,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,qBAAqB,UAAU,EAAE,EAAE,GAAG,EAAE,gCAAgC,EAAE,CAAC,CAAC,CAAC;QACnH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClD,IAAI,IAAI,CAAC,WAAW,KAAK,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/D,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,IAAI,CAAC,aAAa;QAAE,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,IAAI,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3E,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;IACjC,IAAI,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAExC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,WAAW,GAAG,cAAc,IAAI,0BAA0B,CAAC;QACjE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAChF,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;YACtD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,OAAO,GAAG;oBACd,IAAI,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;oBACzB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;oBACzB,KAAK,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;oBAC3E,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;wBACjE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;wBACpE,CAAC,CAAC,CAAC;oBACL,QAAQ;iBACT,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,6BAA6B,EAAE,CAAC,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AACzB,CAAC,CAAC,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Command } from "commander";
2
+ export declare const initCommand: Command;
3
+ //# sourceMappingURL=init.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,eAAO,MAAM,WAAW,SA4LpB,CAAC"}