@webywz/eval-smoke 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/html-report.d.ts +8 -0
- package/dist/html-report.d.ts.map +1 -0
- package/dist/html-report.js +196 -0
- package/dist/html-report.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/run-cli.d.ts +1 -1
- package/dist/run-cli.d.ts.map +1 -1
- package/dist/run-cli.js +9 -0
- package/dist/run-cli.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ eval-smoke suite.yaml --mock
|
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
```text
|
|
28
|
-
eval-smoke 0.
|
|
28
|
+
eval-smoke 0.5.0 · mock · gpt-4o-mini · smoke · 4 cases
|
|
29
29
|
|
|
30
30
|
pass greeting 0ms
|
|
31
31
|
pass capital-france 0ms
|
|
@@ -45,7 +45,7 @@ eval-smoke 0.4.0 · mock · gpt-4o-mini · smoke · 4 cases
|
|
|
45
45
|
| **Matchers** | `equals`, `contains`, `not_contains`, `regex`, `json`, `min_tokens` / `max_tokens` |
|
|
46
46
|
| **init** | `eval-smoke init` scaffolds a mock-friendly `suite.yaml` |
|
|
47
47
|
| **Filter** | `--only` by id substring or `/regex/` |
|
|
48
|
-
| **CI** | `--junit`, `--github-summary`, exit codes, `--fail-fast`, `--concurrency` |
|
|
48
|
+
| **CI** | `--junit`, `--html`, `--github-summary`, exit codes, `--fail-fast`, `--concurrency` |
|
|
49
49
|
| **Baseline** | `--baseline` / `--update-baseline` catch pass→fail regressions |
|
|
50
50
|
| **Providers** | Mock offline; live OpenAI / Anthropic when keyed |
|
|
51
51
|
|
|
@@ -66,6 +66,7 @@ eval-smoke init
|
|
|
66
66
|
eval-smoke suite.yaml --mock
|
|
67
67
|
eval-smoke suite.yaml --mock --only greeting
|
|
68
68
|
eval-smoke suite.yaml --mock --junit junit.xml
|
|
69
|
+
eval-smoke suite.yaml --mock --html report.html
|
|
69
70
|
eval-smoke examples/suite.yaml --mock --json
|
|
70
71
|
eval-smoke examples/suite.yaml --mock --concurrency 4
|
|
71
72
|
eval-smoke examples/suite.yaml --mock --update-baseline baseline.json
|
|
@@ -138,6 +139,7 @@ eval-smoke init [dir] [--force]
|
|
|
138
139
|
| `--mock` | Offline deterministic runner |
|
|
139
140
|
| `--json` | Machine-readable JSON report |
|
|
140
141
|
| `--junit <path>` | Write JUnit XML (still prints human report unless `--json`) |
|
|
142
|
+
| `--html <path>` | Write self-contained HTML report (offline, no CDN) |
|
|
141
143
|
| `--baseline <path>` | Exit `1` if a baseline-passing case now fails or is missing |
|
|
142
144
|
| `--update-baseline <path>` | Write current report JSON (pretty) to path |
|
|
143
145
|
| `--github-summary [path]` | Append pass/fail Markdown to `$GITHUB_STEP_SUMMARY` or path |
|
|
@@ -184,7 +186,7 @@ if (!report.ok) process.exit(1);
|
|
|
184
186
|
### CI snippet
|
|
185
187
|
|
|
186
188
|
```yaml
|
|
187
|
-
- run: npx @webywz/eval-smoke@0.
|
|
189
|
+
- run: npx @webywz/eval-smoke@0.5.0 prompts/smoke.yaml --mock --junit junit.xml
|
|
188
190
|
- uses: actions/upload-artifact@v4
|
|
189
191
|
if: always()
|
|
190
192
|
with:
|
|
@@ -222,7 +224,7 @@ eval-smoke suite.yaml --mock
|
|
|
222
224
|
| **匹配器** | `equals`、`contains`、`not_contains`、`regex`、`json`、token 上下限 |
|
|
223
225
|
| **init** | `eval-smoke init` 生成可 mock 的 `suite.yaml` |
|
|
224
226
|
| **过滤** | `--only`：id 子串或 `/regex/` |
|
|
225
|
-
| **CI** | `--junit`、`--github-summary`、退出码、`--fail-fast`、`--concurrency` |
|
|
227
|
+
| **CI** | `--junit`、`--html`、`--github-summary`、退出码、`--fail-fast`、`--concurrency` |
|
|
226
228
|
| **基线** | `--baseline` / `--update-baseline` 拦住 pass→fail 回归 |
|
|
227
229
|
| **提供商** | Mock 离线；有 key 时走 OpenAI / Anthropic |
|
|
228
230
|
|
|
@@ -243,6 +245,7 @@ eval-smoke init
|
|
|
243
245
|
eval-smoke suite.yaml --mock
|
|
244
246
|
eval-smoke suite.yaml --mock --only greeting
|
|
245
247
|
eval-smoke suite.yaml --mock --junit junit.xml
|
|
248
|
+
eval-smoke suite.yaml --mock --html report.html
|
|
246
249
|
eval-smoke examples/suite.yaml --mock --json
|
|
247
250
|
eval-smoke examples/suite.yaml --mock --concurrency 4
|
|
248
251
|
eval-smoke examples/suite.yaml --mock --update-baseline baseline.json
|
|
@@ -302,6 +305,7 @@ cases:
|
|
|
302
305
|
| `--mock` | 离线确定性运行 |
|
|
303
306
|
| `--json` | JSON 报告 |
|
|
304
307
|
| `--junit <path>` | 写出 JUnit XML（仍打印人类可读报告，除非同时 `--json`） |
|
|
308
|
+
| `--html <path>` | 写入自包含 HTML 报告（离线、无 CDN） |
|
|
305
309
|
| `--baseline <path>` | 基线里曾通过的用例若失败或缺失则退出 `1` |
|
|
306
310
|
| `--update-baseline <path>` | 把当前报告 JSON（pretty）写入路径 |
|
|
307
311
|
| `--github-summary [path]` | 追加通过/失败 Markdown 到 `$GITHUB_STEP_SUMMARY` 或指定路径 |
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Report } from "./types.js";
|
|
2
|
+
/** Escape text for safe insertion into HTML text/attr contexts. */
|
|
3
|
+
export declare function escapeHtml(text: string): string;
|
|
4
|
+
/**
|
|
5
|
+
* Self-contained dark HTML report for an eval-smoke run (inline CSS+JS, no CDN).
|
|
6
|
+
*/
|
|
7
|
+
export declare function formatHtmlReport(report: Report): string;
|
|
8
|
+
//# sourceMappingURL=html-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-report.d.ts","sourceRoot":"","sources":["../src/html-report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAC;AAElE,mEAAmE;AACnE,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO/C;AA8CD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAgJvD"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/** Escape text for safe insertion into HTML text/attr contexts. */
|
|
2
|
+
export function escapeHtml(text) {
|
|
3
|
+
return text
|
|
4
|
+
.replace(/&/g, "&")
|
|
5
|
+
.replace(/</g, "<")
|
|
6
|
+
.replace(/>/g, ">")
|
|
7
|
+
.replace(/"/g, """)
|
|
8
|
+
.replace(/'/g, "'");
|
|
9
|
+
}
|
|
10
|
+
function truncate(text, max = 400) {
|
|
11
|
+
const compact = text.replace(/\s+/g, " ").trim();
|
|
12
|
+
if (compact.length <= max)
|
|
13
|
+
return compact;
|
|
14
|
+
return `${compact.slice(0, max - 1)}…`;
|
|
15
|
+
}
|
|
16
|
+
function failedChecks(result) {
|
|
17
|
+
return (result.checks ?? []).filter((check) => !check.ok);
|
|
18
|
+
}
|
|
19
|
+
function statusLabel(result) {
|
|
20
|
+
if (result.skipped)
|
|
21
|
+
return "skip";
|
|
22
|
+
return result.ok ? "pass" : "fail";
|
|
23
|
+
}
|
|
24
|
+
function statusClass(result) {
|
|
25
|
+
if (result.skipped)
|
|
26
|
+
return "skip";
|
|
27
|
+
return result.ok ? "pass" : "fail";
|
|
28
|
+
}
|
|
29
|
+
function failureDetails(result) {
|
|
30
|
+
const parts = [];
|
|
31
|
+
if (result.error)
|
|
32
|
+
parts.push(result.error);
|
|
33
|
+
for (const check of failedChecks(result)) {
|
|
34
|
+
const detail = check.detail ?? "failed";
|
|
35
|
+
parts.push(`${check.name}: ${detail}`);
|
|
36
|
+
if (check.expected != null && !detail.includes("expected:")) {
|
|
37
|
+
parts.push(`expected: ${truncate(check.expected, 200)}`);
|
|
38
|
+
}
|
|
39
|
+
if (check.actual != null && !detail.includes("actual:")) {
|
|
40
|
+
parts.push(`actual: ${truncate(check.actual, 200)}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
if (result.output &&
|
|
44
|
+
!result.ok &&
|
|
45
|
+
failedChecks(result).every((c) => c.name !== "equals")) {
|
|
46
|
+
parts.push(`output: ${truncate(result.output)}`);
|
|
47
|
+
}
|
|
48
|
+
if (!parts.length)
|
|
49
|
+
return "";
|
|
50
|
+
return `<details><summary>details</summary><pre>${escapeHtml(parts.join("\n"))}</pre></details>`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Self-contained dark HTML report for an eval-smoke run (inline CSS+JS, no CDN).
|
|
54
|
+
*/
|
|
55
|
+
export function formatHtmlReport(report) {
|
|
56
|
+
const suite = escapeHtml(report.suite ?? "suite");
|
|
57
|
+
const version = escapeHtml(report.version);
|
|
58
|
+
const runner = escapeHtml(report.runner);
|
|
59
|
+
const model = escapeHtml(report.model);
|
|
60
|
+
const okClass = report.ok ? "ok" : "bad";
|
|
61
|
+
const rows = report.cases
|
|
62
|
+
.map((result) => {
|
|
63
|
+
const st = statusClass(result);
|
|
64
|
+
const details = failureDetails(result);
|
|
65
|
+
return `<tr class="${st}">
|
|
66
|
+
<td><span class="badge ${st}">${statusLabel(result)}</span></td>
|
|
67
|
+
<td><code>${escapeHtml(result.id)}</code></td>
|
|
68
|
+
<td class="num">${result.duration_ms}ms</td>
|
|
69
|
+
<td class="detail">${details}</td>
|
|
70
|
+
</tr>`;
|
|
71
|
+
})
|
|
72
|
+
.join("\n");
|
|
73
|
+
return `<!DOCTYPE html>
|
|
74
|
+
<html lang="en">
|
|
75
|
+
<head>
|
|
76
|
+
<meta charset="utf-8"/>
|
|
77
|
+
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
78
|
+
<title>eval-smoke · ${suite}</title>
|
|
79
|
+
<style>
|
|
80
|
+
:root {
|
|
81
|
+
--bg: #0a0a0a;
|
|
82
|
+
--panel: #111111;
|
|
83
|
+
--border: #222222;
|
|
84
|
+
--text: #eeeeee;
|
|
85
|
+
--muted: #888888;
|
|
86
|
+
--pass: #7ee787;
|
|
87
|
+
--fail: #ff7b72;
|
|
88
|
+
--skip: #d2a8ff;
|
|
89
|
+
--font: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
|
90
|
+
}
|
|
91
|
+
* { box-sizing: border-box; }
|
|
92
|
+
html, body {
|
|
93
|
+
margin: 0; padding: 0;
|
|
94
|
+
background: var(--bg); color: var(--text);
|
|
95
|
+
font: 13px/1.45 var(--font);
|
|
96
|
+
}
|
|
97
|
+
body { padding: 24px 20px 48px; max-width: 960px; margin: 0 auto; }
|
|
98
|
+
header.top {
|
|
99
|
+
border: 1px solid var(--border); background: var(--panel);
|
|
100
|
+
padding: 16px 18px; margin-bottom: 16px;
|
|
101
|
+
}
|
|
102
|
+
header.top h1 { margin: 0 0 6px; font-size: 15px; font-weight: 600; }
|
|
103
|
+
header.top .meta { color: var(--muted); font-size: 12px; }
|
|
104
|
+
header.top .summary { margin-top: 10px; font-size: 13px; }
|
|
105
|
+
header.top.ok .summary { color: var(--pass); }
|
|
106
|
+
header.top.bad .summary { color: var(--fail); }
|
|
107
|
+
.stats {
|
|
108
|
+
display: grid; grid-template-columns: repeat(4, minmax(0, 1fr));
|
|
109
|
+
gap: 8px; margin: 0 0 18px;
|
|
110
|
+
}
|
|
111
|
+
.stat {
|
|
112
|
+
border: 1px solid var(--border); background: var(--panel);
|
|
113
|
+
padding: 10px 12px;
|
|
114
|
+
}
|
|
115
|
+
.stat .label { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: 0.06em; }
|
|
116
|
+
.stat .value { margin-top: 4px; font-size: 16px; }
|
|
117
|
+
table {
|
|
118
|
+
width: 100%; border-collapse: collapse;
|
|
119
|
+
border: 1px solid var(--border); background: var(--panel);
|
|
120
|
+
}
|
|
121
|
+
th, td {
|
|
122
|
+
text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border);
|
|
123
|
+
vertical-align: top;
|
|
124
|
+
}
|
|
125
|
+
th { color: var(--muted); font-weight: 500; font-size: 11px; text-transform: uppercase; }
|
|
126
|
+
td.num { text-align: right; color: var(--muted); white-space: nowrap; }
|
|
127
|
+
.badge {
|
|
128
|
+
display: inline-block; min-width: 3.2em; text-align: center;
|
|
129
|
+
padding: 1px 6px; border: 1px solid var(--border);
|
|
130
|
+
}
|
|
131
|
+
.badge.pass { color: var(--pass); border-color: #1e3d28; }
|
|
132
|
+
.badge.fail { color: var(--fail); border-color: #5a1d1d; }
|
|
133
|
+
.badge.skip { color: var(--skip); border-color: #3a2a55; }
|
|
134
|
+
tr.fail { background: #140a0a; }
|
|
135
|
+
details { margin-top: 4px; }
|
|
136
|
+
summary { cursor: pointer; color: var(--muted); user-select: none; }
|
|
137
|
+
pre {
|
|
138
|
+
margin: 6px 0 0; padding: 10px; overflow: auto;
|
|
139
|
+
background: #050505; border: 1px solid var(--border);
|
|
140
|
+
white-space: pre-wrap; word-break: break-word; color: #d8d8d8;
|
|
141
|
+
}
|
|
142
|
+
.toolbar { margin: 0 0 12px; display: flex; gap: 8px; }
|
|
143
|
+
.toolbar button {
|
|
144
|
+
background: var(--panel); color: var(--text); border: 1px solid var(--border);
|
|
145
|
+
padding: 6px 10px; font: inherit; cursor: pointer;
|
|
146
|
+
}
|
|
147
|
+
.toolbar button:hover { border-color: #444; }
|
|
148
|
+
footer {
|
|
149
|
+
margin-top: 28px; color: var(--muted); font-size: 11px;
|
|
150
|
+
border-top: 1px solid var(--border); padding-top: 12px;
|
|
151
|
+
}
|
|
152
|
+
code { font: inherit; }
|
|
153
|
+
@media (max-width: 640px) {
|
|
154
|
+
.stats { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
|
155
|
+
}
|
|
156
|
+
</style>
|
|
157
|
+
</head>
|
|
158
|
+
<body>
|
|
159
|
+
<header class="top ${okClass}">
|
|
160
|
+
<h1>eval-smoke ${version}</h1>
|
|
161
|
+
<div class="meta">${runner} · ${model} · ${suite} · ${report.cases.length} cases</div>
|
|
162
|
+
<div class="summary">${report.passed} passed · ${report.failed} failed${report.skipped ? ` · ${report.skipped} skipped` : ""} · ${report.duration_ms}ms</div>
|
|
163
|
+
</header>
|
|
164
|
+
<section class="stats" aria-label="stats">
|
|
165
|
+
<div class="stat"><div class="label">passed</div><div class="value">${report.passed}</div></div>
|
|
166
|
+
<div class="stat"><div class="label">failed</div><div class="value">${report.failed}</div></div>
|
|
167
|
+
<div class="stat"><div class="label">skipped</div><div class="value">${report.skipped}</div></div>
|
|
168
|
+
<div class="stat"><div class="label">duration</div><div class="value">${report.duration_ms}ms</div></div>
|
|
169
|
+
</section>
|
|
170
|
+
<div class="toolbar">
|
|
171
|
+
<button type="button" id="expand-all">Expand failures</button>
|
|
172
|
+
<button type="button" id="collapse-all">Collapse</button>
|
|
173
|
+
</div>
|
|
174
|
+
<table>
|
|
175
|
+
<thead><tr><th>Status</th><th>Case</th><th>Duration</th><th>Details</th></tr></thead>
|
|
176
|
+
<tbody>
|
|
177
|
+
${rows || `<tr><td colspan="4" style="color:var(--muted)">no cases</td></tr>`}
|
|
178
|
+
</tbody>
|
|
179
|
+
</table>
|
|
180
|
+
<footer>Generated by eval-smoke · self-contained · works offline (file://)</footer>
|
|
181
|
+
<script>
|
|
182
|
+
(function () {
|
|
183
|
+
var expand = document.getElementById("expand-all");
|
|
184
|
+
var collapse = document.getElementById("collapse-all");
|
|
185
|
+
function setAll(open) {
|
|
186
|
+
document.querySelectorAll("details").forEach(function (d) { d.open = open; });
|
|
187
|
+
}
|
|
188
|
+
if (expand) expand.addEventListener("click", function () { setAll(true); });
|
|
189
|
+
if (collapse) collapse.addEventListener("click", function () { setAll(false); });
|
|
190
|
+
})();
|
|
191
|
+
</script>
|
|
192
|
+
</body>
|
|
193
|
+
</html>
|
|
194
|
+
`;
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=html-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-report.js","sourceRoot":"","sources":["../src/html-report.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,IAAI;SACR,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAG,GAAG,GAAG;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC;IAC1C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;AACzC,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB;IACtC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,WAAW,CAAC,MAAkB;IACrC,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC;IAClC,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,SAAS,WAAW,CAAC,MAAkB;IACrC,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC;IAClC,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,SAAS,cAAc,CAAC,MAAkB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,QAAQ,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACxD,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IACD,IACE,MAAM,CAAC,MAAM;QACb,CAAC,MAAM,CAAC,EAAE;QACV,YAAY,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,EACtD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,OAAO,2CAA2C,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,kBAAkB,CAAC;AACnG,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC7C,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IAEzC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK;SACtB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,cAAc,EAAE;2BACF,EAAE,KAAK,WAAW,CAAC,MAAM,CAAC;cACvC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBACf,MAAM,CAAC,WAAW;uBACf,OAAO;MACxB,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;;;sBAKa,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAiFN,OAAO;mBACT,OAAO;sBACJ,MAAM,MAAM,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM;yBAClD,MAAM,CAAC,MAAM,aAAa,MAAM,CAAC,MAAM,UAC5D,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EACpD,MAAM,MAAM,CAAC,WAAW;;;wEAG8C,MAAM,CAAC,MAAM;wEACb,MAAM,CAAC,MAAM;yEACZ,MAAM,CAAC,OAAO;0EACb,MAAM,CAAC,WAAW;;;;;;;;;EAS1F,IAAI,IAAI,mEAAmE;;;;;;;;;;;;;;;;;CAiB5E,CAAC;AACF,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export { formatJUnit } from "./junit.js";
|
|
|
4
4
|
export { evaluateCase, evaluateExpect, equalsDiff, normalizeEquals } from "./matchers.js";
|
|
5
5
|
export { createComplete, mockComplete, resolveProvider } from "./providers.js";
|
|
6
6
|
export { formatReport } from "./report.js";
|
|
7
|
+
export { formatHtmlReport, escapeHtml } from "./html-report.js";
|
|
7
8
|
export { runSuite } from "./run.js";
|
|
8
9
|
export { estimateTokens } from "./tokens.js";
|
|
9
10
|
export { VERSION } from "./version.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,IAAI,EACJ,UAAU,EACV,SAAS,EACT,WAAW,EACX,UAAU,EACV,eAAe,EACf,MAAM,EACN,UAAU,EACV,YAAY,EACZ,MAAM,EACN,UAAU,EACV,KAAK,GACN,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,IAAI,EACJ,UAAU,EACV,SAAS,EACT,WAAW,EACX,UAAU,EACV,eAAe,EACf,MAAM,EACN,UAAU,EACV,YAAY,EACZ,MAAM,EACN,UAAU,EACV,KAAK,GACN,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -4,6 +4,7 @@ export { formatJUnit } from "./junit.js";
|
|
|
4
4
|
export { evaluateCase, evaluateExpect, equalsDiff, normalizeEquals } from "./matchers.js";
|
|
5
5
|
export { createComplete, mockComplete, resolveProvider } from "./providers.js";
|
|
6
6
|
export { formatReport } from "./report.js";
|
|
7
|
+
export { formatHtmlReport, escapeHtml } from "./html-report.js";
|
|
7
8
|
export { runSuite } from "./run.js";
|
|
8
9
|
export { estimateTokens } from "./tokens.js";
|
|
9
10
|
export { VERSION } from "./version.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAevC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAC1F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAevC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE3E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/run-cli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Suite } from "./types.js";
|
|
2
|
-
export declare const HELP = "eval-smoke \u2014 one-minute smoke evals for prompts and models\n\nUsage:\n eval-smoke <suite.yaml|suite.json> [options]\n eval-smoke init [dir] [--force]\n\nOptions:\n --mock Offline runner with deterministic fake answers\n --json Machine-readable JSON report\n --junit <path> Write JUnit XML report to path (still prints human report unless --json)\n --only <pattern> Run cases whose id contains pattern, or matches /regex/\n --fail-fast Stop after the first failing case\n --concurrency <n> Run up to N cases in parallel (default: 1; ignored with --fail-fast)\n --timeout <ms> Per-case timeout (default: 30000)\n --model <name> Override suite model\n --provider <name> openai | anthropic | mock\n --baseline <path> Fail if any baseline-passing case now fails or is missing\n --update-baseline <path> Write current report JSON to path (pretty)\n --github-summary [path] Append Markdown to GITHUB_STEP_SUMMARY or path\n --force Overwrite existing suite.yaml when using init\n --no-color Disable ANSI colors (also respects NO_COLOR)\n -h, --help Show this help\n -v, --version Show version\n\nExit codes:\n 0 all cases passed (and baseline clean when --baseline is set)\n 1 one or more cases failed, or baseline regression\n 2 usage or configuration error\n\nExamples:\n eval-smoke init\n eval-smoke init ./prompts --force\n eval-smoke examples/suite.yaml --mock\n eval-smoke examples/suite.yaml --mock --only greeting\n eval-smoke examples/suite.yaml --mock --only '/^capital/'\n eval-smoke examples/suite.yaml --mock --junit junit.xml\n eval-smoke examples/suite.yaml --json --fail-fast\n eval-smoke examples/suite.yaml --mock --concurrency 4\n eval-smoke examples/suite.yaml --mock --update-baseline baseline.json\n eval-smoke examples/suite.yaml --mock --baseline baseline.json\n eval-smoke examples/suite.yaml --mock --github-summary\n OPENAI_API_KEY=sk-... eval-smoke examples/suite.yaml\n";
|
|
2
|
+
export declare const HELP = "eval-smoke \u2014 one-minute smoke evals for prompts and models\n\nUsage:\n eval-smoke <suite.yaml|suite.json> [options]\n eval-smoke init [dir] [--force]\n\nOptions:\n --mock Offline runner with deterministic fake answers\n --json Machine-readable JSON report\n --junit <path> Write JUnit XML report to path (still prints human report unless --json)\n --html <path> Write self-contained HTML report to path (offline, no CDN)\n --only <pattern> Run cases whose id contains pattern, or matches /regex/\n --fail-fast Stop after the first failing case\n --concurrency <n> Run up to N cases in parallel (default: 1; ignored with --fail-fast)\n --timeout <ms> Per-case timeout (default: 30000)\n --model <name> Override suite model\n --provider <name> openai | anthropic | mock\n --baseline <path> Fail if any baseline-passing case now fails or is missing\n --update-baseline <path> Write current report JSON to path (pretty)\n --github-summary [path] Append Markdown to GITHUB_STEP_SUMMARY or path\n --force Overwrite existing suite.yaml when using init\n --no-color Disable ANSI colors (also respects NO_COLOR)\n -h, --help Show this help\n -v, --version Show version\n\nExit codes:\n 0 all cases passed (and baseline clean when --baseline is set)\n 1 one or more cases failed, or baseline regression\n 2 usage or configuration error\n\nExamples:\n eval-smoke init\n eval-smoke init ./prompts --force\n eval-smoke examples/suite.yaml --mock\n eval-smoke examples/suite.yaml --mock --only greeting\n eval-smoke examples/suite.yaml --mock --only '/^capital/'\n eval-smoke examples/suite.yaml --mock --junit junit.xml\n eval-smoke examples/suite.yaml --mock --html report.html\n eval-smoke examples/suite.yaml --json --fail-fast\n eval-smoke examples/suite.yaml --mock --concurrency 4\n eval-smoke examples/suite.yaml --mock --update-baseline baseline.json\n eval-smoke examples/suite.yaml --mock --baseline baseline.json\n eval-smoke examples/suite.yaml --mock --github-summary\n OPENAI_API_KEY=sk-... eval-smoke examples/suite.yaml\n";
|
|
3
3
|
export interface CliIo {
|
|
4
4
|
stdout: {
|
|
5
5
|
write(text: string): void;
|
package/dist/run-cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-cli.d.ts","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-cli.d.ts","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"AAaA,OAAO,EAAiC,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AAMvE,eAAO,MAAM,IAAI,+nEA4ChB,CAAC;AAEF,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtC,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;CAChB;AAkCD,iFAAiF;AACjF,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAQ9D;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,CAMtE;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAiBnE;AAQD,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAE,KAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAsIrF"}
|
package/dist/run-cli.js
CHANGED
|
@@ -7,6 +7,7 @@ import { initSuite } from "./init.js";
|
|
|
7
7
|
import { formatJUnit } from "./junit.js";
|
|
8
8
|
import { loadSuite } from "./load-suite.js";
|
|
9
9
|
import { compileRegex } from "./matchers.js";
|
|
10
|
+
import { formatHtmlReport } from "./html-report.js";
|
|
10
11
|
import { formatReport } from "./report.js";
|
|
11
12
|
import { DEFAULT_TIMEOUT_MS, runSuite } from "./run.js";
|
|
12
13
|
import { formatGithubSummary } from "./summary-md.js";
|
|
@@ -24,6 +25,7 @@ Options:
|
|
|
24
25
|
--mock Offline runner with deterministic fake answers
|
|
25
26
|
--json Machine-readable JSON report
|
|
26
27
|
--junit <path> Write JUnit XML report to path (still prints human report unless --json)
|
|
28
|
+
--html <path> Write self-contained HTML report to path (offline, no CDN)
|
|
27
29
|
--only <pattern> Run cases whose id contains pattern, or matches /regex/
|
|
28
30
|
--fail-fast Stop after the first failing case
|
|
29
31
|
--concurrency <n> Run up to N cases in parallel (default: 1; ignored with --fail-fast)
|
|
@@ -50,6 +52,7 @@ Examples:
|
|
|
50
52
|
eval-smoke examples/suite.yaml --mock --only greeting
|
|
51
53
|
eval-smoke examples/suite.yaml --mock --only '/^capital/'
|
|
52
54
|
eval-smoke examples/suite.yaml --mock --junit junit.xml
|
|
55
|
+
eval-smoke examples/suite.yaml --mock --html report.html
|
|
53
56
|
eval-smoke examples/suite.yaml --json --fail-fast
|
|
54
57
|
eval-smoke examples/suite.yaml --mock --concurrency 4
|
|
55
58
|
eval-smoke examples/suite.yaml --mock --update-baseline baseline.json
|
|
@@ -148,6 +151,7 @@ export async function runCli(argv, io = defaultIo()) {
|
|
|
148
151
|
mock: { type: "boolean", default: false },
|
|
149
152
|
json: { type: "boolean", default: false },
|
|
150
153
|
junit: { type: "string" },
|
|
154
|
+
html: { type: "string" },
|
|
151
155
|
only: { type: "string" },
|
|
152
156
|
"fail-fast": { type: "boolean", default: false },
|
|
153
157
|
concurrency: { type: "string" },
|
|
@@ -203,6 +207,7 @@ export async function runCli(argv, io = defaultIo()) {
|
|
|
203
207
|
const json = asBool(values.json);
|
|
204
208
|
const only = asString(values.only);
|
|
205
209
|
const junitPath = asString(values.junit);
|
|
210
|
+
const htmlPath = asString(values.html);
|
|
206
211
|
const baselinePath = asString(values.baseline);
|
|
207
212
|
const updateBaselinePath = asString(values["update-baseline"]);
|
|
208
213
|
const githubSummary = githubSummaryFlag(asString(values["github-summary"]));
|
|
@@ -223,6 +228,10 @@ export async function runCli(argv, io = defaultIo()) {
|
|
|
223
228
|
const out = resolve(io.cwd, junitPath);
|
|
224
229
|
writeFileSync(out, formatJUnit(report), "utf8");
|
|
225
230
|
}
|
|
231
|
+
if (htmlPath != null) {
|
|
232
|
+
const out = resolve(io.cwd, htmlPath);
|
|
233
|
+
writeFileSync(out, formatHtmlReport(report), "utf8");
|
|
234
|
+
}
|
|
226
235
|
if (updateBaselinePath != null) {
|
|
227
236
|
const out = resolve(io.cwd, updateBaselinePath);
|
|
228
237
|
writeBaseline(out, report);
|
package/dist/run-cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-cli.js","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAiC,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,kEAAkE;AAClE,MAAM,mBAAmB,GAAG,UAAU,CAAC;AAEvC,MAAM,CAAC,MAAM,IAAI,GAAG
|
|
1
|
+
{"version":3,"file":"run-cli.js","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAiC,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,kEAAkE;AAClE,MAAM,mBAAmB,GAAG,UAAU,CAAC;AAEvC,MAAM,CAAC,MAAM,IAAI,GAAG;;;;;;;;;;;;;;mDAc+B,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8BpE,CAAC;AAUF,MAAM,SAAS,GAAG,GAAU,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;CACrC,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,GAAuB;IAC3C,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,kBAAkB,CAAC;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,qDAAqD,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAuB;IAC/C,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,0CAA0C,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,GAAG,CAAC;IAC1E,MAAM,IAAI,UAAU,CAAC,+CAA+C,CAAC,CAAC;AACxE,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,SAAS,CAAC,EAAU,EAAE,OAAe;IACnD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC5D,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACjC,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;QACjB,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAY,EAAE,OAAe;IAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IAChF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,UAAU,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;AAC7B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,IAAc;IACvD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,KAAK,KAAK,kBAAkB,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,GAAG,CAAC,IAAI,CAAC,oBAAoB,mBAAmB,EAAE,CAAC,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC;gBACnC,CAAC,IAAI,CAAC,CAAC;YACT,CAAC;YACD,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAuB;IAChD,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,GAAG,KAAK,mBAAmB;QAAE,OAAO,IAAI,CAAC;IAC7C,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAc,EAAE,KAAY,SAAS,EAAE;IAClE,IAAI,MAAoC,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC;YACjB,IAAI,EAAE,0BAA0B,CAAC,IAAI,CAAC;YACtC,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBACzC,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACxB,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBAChD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC/B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC5B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC5B,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACrC,gBAAgB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACpC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBAC1C,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBAC/C,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE;gBACrD,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE;aACzD;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IACvC,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsB,EAAE,CACtD,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,KAAc,EAAW,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC;IAE3D,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;YAC/E,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;YAC5C,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,CAAC,IAAI,WAAW,CAAC,CAAC;YAC5D,OAAO,CAAC,CAAC;QACX,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;YAChC,OAAO,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC3C,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC/D,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAE5E,IAAI,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE;YACnC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACzB,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClD,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACrC,WAAW,EAAE,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAC3D,SAAS,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,GAAG,EAAE,EAAE,CAAC,GAAG;SACZ,CAAC,CAAC;QACH,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACvC,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACtC,aAAa,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;YAChD,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC3B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;YAC7D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC9C,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,EAAE;gBAAE,cAAc,GAAG,IAAI,CAAC;QACvC,CAAC;QAED,MAAM,UAAU,GACd,aAAa,KAAK,SAAS;YACzB,CAAC,CAAC,wBAAwB,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;YACrD,CAAC,CAAC,wBAAwB,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QAC9D,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,mBAAmB,CAAC,UAAU,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACvE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,OAAO,IAAI,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC;QAC5C,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACvD,IAAI,cAAc;YAAE,OAAO,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webywz/eval-smoke",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "One-minute smoke evals for prompts and models
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "One-minute smoke evals for prompts and models — equals/json matchers, diffs, concurrency, JUnit, init.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"eval-smoke": "dist/cli.js"
|