niceeval 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/types.d.ts +72 -6
- package/dist/context/types.d.ts +32 -12
- package/dist/i18n/en.d.ts +54 -0
- package/dist/i18n/zh-CN.d.ts +55 -1
- package/dist/o11y/types.d.ts +16 -2
- package/dist/report/aggregate.d.ts +5 -3
- package/dist/report/aggregate.js +32 -5
- package/dist/report/built-ins/experiment-comparison.d.ts +39 -0
- package/dist/report/built-ins/experiment-comparison.js +119 -0
- package/dist/report/built-ins/index.d.ts +2 -1
- package/dist/report/built-ins/index.js +2 -2
- package/dist/report/components.d.ts +10 -2
- package/dist/report/components.js +3 -3
- package/dist/report/compute.d.ts +11 -18
- package/dist/report/compute.js +68 -66
- package/dist/report/flag.d.ts +16 -1
- package/dist/report/flag.js +19 -1
- package/dist/report/format.d.ts +16 -14
- package/dist/report/format.js +28 -30
- package/dist/report/index.d.ts +5 -4
- package/dist/report/index.js +6 -5
- package/dist/report/locale.d.ts +23 -3
- package/dist/report/locale.js +47 -6
- package/dist/report/metrics.d.ts +13 -1
- package/dist/report/metrics.js +66 -15
- package/dist/report/primitives.d.ts +6 -0
- package/dist/report/react/AttemptList.d.ts +4 -4
- package/dist/report/react/AttemptList.js +8 -10
- package/dist/report/react/EvalList.d.ts +1 -1
- package/dist/report/react/EvalList.js +0 -0
- package/dist/report/react/ExperimentComparison.d.ts +8 -0
- package/dist/report/react/ExperimentComparison.js +11 -0
- package/dist/report/react/ExperimentList.d.ts +4 -2
- package/dist/report/react/ExperimentList.js +57 -7
- package/dist/report/react/MetricScatter.js +6 -16
- package/dist/report/react/chart-math.d.ts +23 -6
- package/dist/report/react/chart-math.js +71 -19
- package/dist/report/react/fixtures.d.ts +3 -3
- package/dist/report/react/fixtures.js +30 -18
- package/dist/report/react/format.d.ts +1 -1
- package/dist/report/react/format.js +1 -1
- package/dist/report/react/index.d.ts +1 -1
- package/dist/report/report.d.ts +5 -1
- package/dist/report/report.js +6 -2
- package/dist/report/text/faces.d.ts +1 -1
- package/dist/report/text/faces.js +100 -61
- package/dist/report/text/table.js +36 -5
- package/dist/report/types.d.ts +40 -34
- package/dist/results/types.d.ts +11 -0
- package/dist/runner/feedback/sink.d.ts +110 -0
- package/dist/runner/types.d.ts +513 -22
- package/dist/sandbox/docker.d.ts +23 -2
- package/dist/sandbox/e2b.d.ts +15 -1
- package/dist/sandbox/errors.d.ts +30 -3
- package/dist/sandbox/io-retry.d.ts +17 -0
- package/dist/sandbox/registry.d.ts +2 -0
- package/dist/sandbox/resolve.d.ts +18 -5
- package/dist/sandbox/retry.d.ts +11 -1
- package/dist/sandbox/types.d.ts +39 -5
- package/dist/sandbox/vercel.d.ts +7 -1
- package/dist/scoring/coverage.d.ts +30 -0
- package/dist/scoring/display.d.ts +21 -0
- package/dist/scoring/display.js +120 -0
- package/dist/scoring/types.d.ts +103 -20
- package/dist/shared/aggregate.d.ts +1 -0
- package/dist/shared/aggregate.js +3 -3
- package/dist/shared/types.d.ts +28 -0
- package/dist/tty-line.d.ts +0 -4
- package/dist/util.d.ts +23 -0
- package/docs-site/zh/concepts/adapter.mdx +24 -6
- package/docs-site/zh/concepts/assert.mdx +11 -10
- package/docs-site/zh/concepts/evals.mdx +7 -6
- package/docs-site/zh/concepts/experiment.mdx +1 -1
- package/docs-site/zh/concepts/overview.mdx +7 -7
- package/docs-site/zh/guides/agent-feedback-loop.mdx +35 -31
- package/docs-site/zh/guides/authoring.mdx +33 -0
- package/docs-site/zh/guides/ci-integration.mdx +23 -12
- package/docs-site/zh/guides/connect-your-agent.mdx +29 -3
- package/docs-site/zh/guides/custom-reports.mdx +29 -34
- package/docs-site/zh/guides/dataset-fanout.mdx +25 -3
- package/docs-site/zh/guides/debug-sandbox.mdx +57 -0
- package/docs-site/zh/guides/debugging.mdx +210 -0
- package/docs-site/zh/guides/experiments.mdx +10 -3
- package/docs-site/zh/guides/fixtures.mdx +3 -1
- package/docs-site/zh/guides/official-adapters.mdx +27 -3
- package/docs-site/zh/guides/publish-report.mdx +30 -16
- package/docs-site/zh/guides/report-components.mdx +49 -37
- package/docs-site/zh/guides/reporters.mdx +2 -2
- package/docs-site/zh/guides/results-data.mdx +42 -8
- package/docs-site/zh/guides/runner.mdx +17 -7
- package/docs-site/zh/guides/sandbox-agent.mdx +57 -7
- package/docs-site/zh/guides/sandbox-providers.mdx +258 -10
- package/docs-site/zh/guides/scoring-guide.mdx +4 -4
- package/docs-site/zh/guides/viewing-results.mdx +85 -41
- package/docs-site/zh/guides/write-experiment.mdx +5 -3
- package/docs-site/zh/guides/write-send.mdx +19 -2
- package/docs-site/zh/index.mdx +1 -1
- package/docs-site/zh/reference/builtin-agents.mdx +27 -0
- package/docs-site/zh/reference/capabilities.mdx +2 -2
- package/docs-site/zh/reference/cli.mdx +35 -9
- package/docs-site/zh/reference/define-agent.mdx +60 -5
- package/docs-site/zh/reference/define-config.mdx +1 -1
- package/docs-site/zh/reference/define-eval.mdx +42 -9
- package/docs-site/zh/reference/events.mdx +2 -2
- package/docs-site/zh/reference/expect.mdx +36 -6
- package/package.json +5 -1
- package/src/agents/ai-sdk-otel.test.ts +1 -0
- package/src/agents/ai-sdk.test.ts +3 -0
- package/src/agents/ai-sdk.ts +3 -0
- package/src/agents/bub-install-spec.test.ts +34 -0
- package/src/agents/bub-install-spec.ts +32 -0
- package/src/agents/bub.ts +31 -32
- package/src/agents/claude-code.test.ts +130 -9
- package/src/agents/claude-code.ts +76 -4
- package/src/agents/codex.test.ts +189 -40
- package/src/agents/codex.ts +155 -14
- package/src/agents/coding-cli-versions.test.ts +15 -0
- package/src/agents/coding-cli-versions.ts +3 -0
- package/src/agents/index.ts +11 -0
- package/src/agents/langgraph.test.ts +204 -0
- package/src/agents/langgraph.ts +495 -0
- package/src/agents/marketplace.ts +85 -0
- package/src/agents/native-config.test.ts +179 -0
- package/src/agents/native-config.ts +267 -0
- package/src/agents/openai-compat.test.ts +1 -0
- package/src/agents/openclaw.test.ts +31 -0
- package/src/agents/openclaw.ts +171 -0
- package/src/agents/plugin-config.test.ts +1 -0
- package/src/agents/sdk-streams.test.ts +79 -0
- package/src/agents/sdk-streams.ts +55 -10
- package/src/agents/skills.test.ts +1 -0
- package/src/agents/streaming.test.ts +3 -9
- package/src/agents/types.ts +73 -6
- package/src/agents/ui-message-stream.test.ts +3 -0
- package/src/cli.ts +411 -108
- package/src/context/context.test.ts +51 -12
- package/src/context/context.ts +161 -29
- package/src/context/session.test.ts +1 -0
- package/src/context/session.ts +114 -6
- package/src/context/types.ts +30 -12
- package/src/define.test.ts +13 -8
- package/src/define.ts +25 -4
- package/src/expect/index.ts +53 -23
- package/src/i18n/en.ts +65 -4
- package/src/i18n/zh-CN.ts +66 -4
- package/src/o11y/cost.test.ts +1 -0
- package/src/o11y/execution-tree.test.ts +1 -20
- package/src/o11y/otlp/mappers/claude-code.test.ts +1 -0
- package/src/o11y/otlp/parse.test.ts +1 -0
- package/src/o11y/otlp/turn-otel.test.ts +1 -0
- package/src/o11y/parsers/bub.test.ts +1 -0
- package/src/o11y/parsers/claude-code.test.ts +1 -34
- package/src/o11y/parsers/openclaw.test.ts +154 -0
- package/src/o11y/parsers/openclaw.ts +310 -0
- package/src/o11y/prices.json +746 -311
- package/src/o11y/tool-names.test.ts +1 -0
- package/src/o11y/types.ts +16 -2
- package/src/report/aggregate.ts +34 -5
- package/src/report/built-in-user-parity.test.tsx +127 -173
- package/src/report/built-ins/experiment-comparison.tsx +179 -0
- package/src/report/built-ins/index.ts +7 -2
- package/src/report/components.tsx +11 -3
- package/src/report/compute.ts +80 -74
- package/src/report/dual-render.test.tsx +222 -91
- package/src/report/flag.ts +30 -2
- package/src/report/format.ts +36 -27
- package/src/report/index.ts +23 -6
- package/src/report/locale.ts +49 -6
- package/src/report/metrics.ts +68 -15
- package/src/report/primitives.tsx +6 -0
- package/src/report/react/AttemptList.tsx +9 -36
- package/src/report/react/EvalList.tsx +0 -0
- package/src/report/react/ExperimentComparison.tsx +68 -0
- package/src/report/react/ExperimentList.tsx +173 -55
- package/src/report/react/MetricScatter.tsx +13 -25
- package/src/report/react/chart-math.test.ts +85 -0
- package/src/report/react/chart-math.ts +101 -22
- package/src/report/react/enhance.js +72 -1
- package/src/report/react/fixtures.ts +34 -21
- package/src/report/react/format.ts +1 -1
- package/src/report/react/index.tsx +0 -1
- package/src/report/react/render.test.tsx +30 -69
- package/src/report/react/styles.css +112 -14
- package/src/report/report.test.ts +308 -105
- package/src/report/report.ts +6 -2
- package/src/report/text/faces.ts +111 -67
- package/src/report/text/table.ts +42 -5
- package/src/report/types.ts +42 -34
- package/src/results/annotated-source.test.ts +62 -9
- package/src/results/annotated-source.ts +64 -6
- package/src/results/attempt-evidence.test.ts +9 -7
- package/src/results/attempt-evidence.ts +15 -8
- package/src/results/attempt-source.ts +6 -3
- package/src/results/copy.ts +145 -55
- package/src/results/host-equivalence.test.ts +11 -9
- package/src/results/index.ts +2 -0
- package/src/results/locator.test.ts +1 -22
- package/src/results/open.ts +7 -1
- package/src/results/publish.ts +149 -0
- package/src/results/results.test.ts +85 -51
- package/src/results/truncate.ts +90 -0
- package/src/results/types.ts +7 -0
- package/src/results/writer.ts +31 -13
- package/src/runner/attempt.test.ts +138 -7
- package/src/runner/attempt.ts +603 -104
- package/src/runner/discover.test.ts +47 -0
- package/src/runner/discover.ts +36 -2
- package/src/runner/eval-source.test.ts +1 -27
- package/src/runner/feedback/agent.test.ts +504 -0
- package/src/runner/feedback/agent.ts +409 -0
- package/src/runner/feedback/ci.test.ts +562 -0
- package/src/runner/feedback/ci.ts +401 -0
- package/src/runner/feedback/coordinator.test.ts +317 -0
- package/src/runner/feedback/coordinator.ts +397 -0
- package/src/runner/feedback/failure.ts +40 -0
- package/src/runner/feedback/human.test.ts +616 -0
- package/src/runner/feedback/human.ts +535 -0
- package/src/runner/feedback/index.ts +66 -0
- package/src/runner/feedback/io.ts +78 -0
- package/src/runner/feedback/profile.test.ts +50 -0
- package/src/runner/feedback/profile.ts +58 -0
- package/src/runner/feedback/reducer.test.ts +395 -0
- package/src/runner/feedback/reducer.ts +260 -0
- package/src/runner/feedback/renderer.ts +82 -0
- package/src/runner/feedback/sink.ts +203 -0
- package/src/runner/feedback/testing.ts +106 -0
- package/src/runner/ledger.test.ts +230 -0
- package/src/runner/ledger.ts +329 -0
- package/src/runner/report.test.ts +128 -3
- package/src/runner/report.ts +33 -9
- package/src/runner/reporters/artifacts.ts +8 -2
- package/src/runner/reporters/braintrust.test.ts +8 -7
- package/src/runner/reporters/braintrust.ts +9 -2
- package/src/runner/reporters/index.ts +2 -2
- package/src/runner/reporters/json.test.ts +162 -0
- package/src/runner/reporters/json.ts +35 -8
- package/src/runner/reporters/shared.ts +1 -5
- package/src/runner/run.test.ts +760 -3
- package/src/runner/run.ts +242 -36
- package/src/runner/sandbox-prep.ts +3 -42
- package/src/runner/timing.ts +158 -0
- package/src/runner/types.ts +518 -22
- package/src/sandbox/checkpoint.test.ts +55 -0
- package/src/sandbox/checkpoint.ts +29 -8
- package/src/sandbox/cli-commands.ts +407 -0
- package/src/sandbox/docker.ts +115 -16
- package/src/sandbox/e2b-agent-template.test.ts +56 -0
- package/src/sandbox/e2b-agent-template.ts +94 -0
- package/src/sandbox/e2b.ts +74 -9
- package/src/sandbox/errors.ts +111 -4
- package/src/sandbox/index.ts +2 -0
- package/src/sandbox/io-retry.test.ts +58 -0
- package/src/sandbox/io-retry.ts +45 -0
- package/src/sandbox/keep-registry.test.ts +86 -0
- package/src/sandbox/keep-registry.ts +142 -0
- package/src/sandbox/keep.ts +178 -0
- package/src/sandbox/paths.test.ts +1 -0
- package/src/sandbox/paths.ts +19 -8
- package/src/sandbox/registry.ts +20 -3
- package/src/sandbox/resolve.ts +76 -11
- package/src/sandbox/retry.test.ts +70 -0
- package/src/sandbox/retry.ts +46 -4
- package/src/sandbox/types.ts +44 -6
- package/src/sandbox/vercel.ts +43 -20
- package/src/scoring/collector.ts +60 -17
- package/src/scoring/coverage.ts +95 -0
- package/src/scoring/diff.ts +81 -0
- package/src/scoring/display.test.ts +121 -0
- package/src/scoring/display.ts +133 -0
- package/src/scoring/evidence.test.ts +189 -0
- package/src/scoring/judge.test.ts +142 -0
- package/src/scoring/judge.ts +15 -18
- package/src/scoring/scoped.ts +217 -50
- package/src/scoring/types.ts +117 -20
- package/src/scoring/verdict.ts +16 -4
- package/src/shared/aggregate.ts +3 -2
- package/src/shared/types.ts +31 -0
- package/src/show/compose.ts +2 -2
- package/src/show/index.ts +29 -16
- package/src/show/render.ts +626 -308
- package/src/show/show.test.ts +251 -36
- package/src/tty-line.ts +8 -26
- package/src/util.test.ts +1 -0
- package/src/util.ts +41 -0
- package/src/view/app/components/AttemptModal.tsx +153 -2
- package/src/view/app/components/CodeView.tsx +32 -11
- package/src/view/app/components/CopyControls.tsx +2 -2
- package/src/view/app/i18n.ts +6 -0
- package/src/view/app/lib/attempt-route.test.ts +1 -0
- package/src/view/app/lib/verdict.ts +7 -9
- package/src/view/artifact-serving.test.ts +2 -1
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +17 -17
- package/src/view/data.test.ts +2 -1
- package/src/view/data.ts +17 -7
- package/src/view/index.ts +12 -1
- package/src/view/server.ts +2 -0
- package/src/view/shared/types.ts +1 -1
- package/src/view/styles.css +3 -0
- package/src/view/view-report.test.ts +11 -10
- package/dist/o11y/execution-tree.d.ts +0 -103
- package/dist/o11y/otlp/select.d.ts +0 -22
- package/dist/report/built-ins/cost-pass-rate-comparison.d.ts +0 -1
- package/dist/report/built-ins/cost-pass-rate-comparison.js +0 -17
- package/dist/results/annotated-source.d.ts +0 -61
- package/dist/results/attempt-evidence.d.ts +0 -69
- package/dist/results/attempt-source.d.ts +0 -15
- package/src/report/built-ins/cost-pass-rate-comparison.tsx +0 -23
- package/src/runner/reporters/console.ts +0 -70
- package/src/runner/reporters/live.test.ts +0 -56
- package/src/runner/reporters/live.ts +0 -247
- package/src/runner/reporters/quiet.test.ts +0 -66
- package/src/runner/reporters/quiet.ts +0 -49
- package/src/runner/reporters/table.ts +0 -277
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
// CI profile renderer(见 docs/feature/experiments/cli.md「CI 怎么用」)。
|
|
2
|
+
//
|
|
3
|
+
// 目标读者是 CI job 日志和读日志的人/annotation adapter,不是交互终端:
|
|
4
|
+
// - 「run:start 之后」的一切正常事件(start/progress/failed/errored/diagnostic/budget_exhausted/
|
|
5
|
+
// interrupted/reporter_error/result/json/junit/snapshots)全部走 `stdout` 一个 sink——不拆到
|
|
6
|
+
// stderr,因为大多数 CI runner 把两个 OS stream 分开缓冲/采集,交错写会打乱真实发生顺序
|
|
7
|
+
// (checklist 第一条)。`stderr` 只保留给 run 尚未建立前的 argv/config 错误,那些错误发生在
|
|
8
|
+
// coordinator/renderer 存在之前,根本不经过这个模块——所以这里没有任何写 `io.stderr` 的分支,
|
|
9
|
+
// 不是遗漏。
|
|
10
|
+
// - 固定 ASCII `niceeval: <word> key=value ...` 行,字段名是英文字面量,不查 i18n 字典、不读
|
|
11
|
+
// `NICEEVAL_LANG`(checklist 第二条)——这与 human.ts 形成对照:human 走 `t()` 因为它是给人看的,
|
|
12
|
+
// 这里每一处文案都是硬编码字符串。
|
|
13
|
+
// - `start` 在 "plan" 永久事件到达时同步打印,不需要像 agent.ts 那样等第一次 tick——CI 的 start
|
|
14
|
+
// 行只有 `total/configs/concurrency/reused` 四个字段(见 cli.md 字面例子),这些在 "plan" 那一刻
|
|
15
|
+
// 全部已知(不像 agent 的 progress 行还带 `running/queued`,那两个要等调度真的把 attempt 派发
|
|
16
|
+
// 出去才有意义)。
|
|
17
|
+
// - 之后只有 failed/errored/diagnostic 类永久事件立即追加,其余状态变化只在连续 60 秒没有任何
|
|
18
|
+
// 永久事件时才追加一条 progress heartbeat(checklist 第三条);任何永久事件(不止 diagnostic,
|
|
19
|
+
// 也包括 failure/budget-exhausted/interrupted/reporter-error)都重置这个 60 秒时钟——与
|
|
20
|
+
// agent.ts 对「warning」的理解一致,这里同样用「lastCheckpointAtMs 被 appendDurable 无条件
|
|
21
|
+
// 更新」来实现,不单独区分「哪种永久事件才算数」。
|
|
22
|
+
// - 不实现 `clearDynamic`/`redrawDynamic`/`activity`/`onLifecycle`:CI 没有动态区域,不展示
|
|
23
|
+
// active phase、不逐次输出 provisioning retry/backoff、不逐条打印 passed attempt——这些目标
|
|
24
|
+
// 行为由「不实现对应可选钩子」天然满足(见 renderer.ts 的接口注释),不需要在这里写
|
|
25
|
+
// `if (profile === "ci")` 分支。
|
|
26
|
+
//
|
|
27
|
+
// 与 human.ts / agent.ts 刻意不共享任何内部函数/类型(哪怕字面意思相同,如 group 推导、
|
|
28
|
+
// 转义、verdict 优先级)——三个 renderer 各自独立成立,不应该因为共享一个私有 helper 而在未来
|
|
29
|
+
// 被迫同步重构(与 agent.ts 顶部注释同一原则)。
|
|
30
|
+
//
|
|
31
|
+
// 为什么 errored 的 checkpoint 行不拆 `code=`/`message=`:
|
|
32
|
+
// 与 agent.ts 面对的是同一个 `FailureNotice`——执行错误只有一个整句 `reason` 字段,没有独立的
|
|
33
|
+
// `code` 字段;failed assertion 则携带 assertion/matcher/expected/received 等结构化字段。
|
|
34
|
+
// cli.md「CI 怎么用」给出的 `errored` 例子(`code=sandbox-rate-limit message="..."`)和
|
|
35
|
+
// 「timeout、budget 与基础设施错误」给出的另一个例子(`kind=timeout timeout_ms=60000`)彼此
|
|
36
|
+
// 字段名都不一致,且都要求解析/猜测 `reason` 文案的隐含格式才能拆出来——这正是顶层「不接受」
|
|
37
|
+
// 清单禁止的「Agent/CI 解析 human 文案」。这里改为只输出类型里已有的结构化字段
|
|
38
|
+
// (locator/eval/experiment/phase/reason),`reason` 作为整句、不做二次拆分的字段直接输出
|
|
39
|
+
// (与「立即追加」给出的 `failed` 例子一致:那一行确实是把整句塞进 `reason=` 一个字段)。
|
|
40
|
+
//
|
|
41
|
+
// 为什么 `elapsed=`/`duration=` 统一用「原始秒数,不做分钟换算」:
|
|
42
|
+
// cli.md「CI 怎么用」同一段字面例子里,`elapsed=60s`/`elapsed=120s`(均已过 1 分钟)与
|
|
43
|
+
// `duration=128s`(同样已过 2 分钟)三处全部是原始秒数、不换算成 `Xm YYs`;但另一处独立的
|
|
44
|
+
// budget 例子给出 `duration=18m02s`——同一份文档对「多长该不该换算」自相矛盾(与 Stage D/E
|
|
45
|
+
// 已经记录的其它 cli.md 例子内部不一致是同一类问题)。这里选择与「同一个例子块内」三处数字
|
|
46
|
+
// 一致的算法(原始秒数),不为了迁就另一个独立例子引入 `Xm YYs` 换算——引入换算会让
|
|
47
|
+
// `elapsed=120s` 变成 `elapsed=2m00s`,反而背离了给出最多字面文本的那个例子。集成测试
|
|
48
|
+
// (ci.test.ts)覆盖 budget-incomplete 场景时按这个算法断言数字,不追求复现 "18m02s" 这个
|
|
49
|
+
// 孤立字符串。
|
|
50
|
+
//
|
|
51
|
+
// 为什么 `DurableFeedbackEvent` 的 "saved" 变体多了 `json`/`junit` 两个字段(见 ../types.ts):
|
|
52
|
+
// cli.md 给出的 result 收尾字面例子是三条独立的行——`json=.../ci-summary.json`、
|
|
53
|
+
// `junit=.../junit.xml`、`snapshots=.../<3 snapshots>`——分别对应「用户显式要的聚合报告是否
|
|
54
|
+
// 写出成功」与「快照落盘在哪」两类不同信息。原有 "saved" 事件只有一个扁平 `paths: string[]`,
|
|
55
|
+
// 没有字段能让这个 renderer 区分「这条路径是 JSON 报告」还是「这是一个快照目录」,逼得只能靠
|
|
56
|
+
// 猜文件后缀反推——这正是要避免的隐式解析。这里改为在 `paths`(继续只表示快照结果路径,
|
|
57
|
+
// human.ts/agent.ts 的既有用法不变)之外新增两个可选字段,只在真的写出对应文件时才有值。
|
|
58
|
+
// `FeedbackCoordinator.finish()` 的入参同步加了这两个可选字段并原样转发——纯新增、不改变
|
|
59
|
+
// 任何已有调用点的类型检查(human/agent 的测试与实现都不读这两个新字段)。
|
|
60
|
+
|
|
61
|
+
import type { FeedbackRenderer } from "./renderer.ts";
|
|
62
|
+
import type { FeedbackIO } from "./io.ts";
|
|
63
|
+
import type { DurableFeedbackEvent, RunCompletion, RunFeedbackState, RunSummary } from "../types.ts";
|
|
64
|
+
|
|
65
|
+
/** failed/errored 立即展开上限(cli.md「'立即追加'也必须有上限」表:ci 前 50 条;完整清单
|
|
66
|
+
* 由 JSON/JUnit 保存,checklist 第四条)。 */
|
|
67
|
+
const CI_FAILURE_CAP = 50;
|
|
68
|
+
/** 连续无永久事件多久才追加一条 progress heartbeat(checklist 第三条:「连续 60 秒无永久事件
|
|
69
|
+
* 才 heartbeat」)。 */
|
|
70
|
+
const CI_HEARTBEAT_IDLE_MS = 60_000;
|
|
71
|
+
|
|
72
|
+
export interface CiRendererOptions {
|
|
73
|
+
io: FeedbackIO;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* 创建 ci profile 的 `FeedbackRenderer`。只用 `io.stdout` 写文本——checklist 第一条要求
|
|
78
|
+
* 「CI 正常事件全部走一个 stdout sink」,这个模块里没有任何 `io.stderr.write` 调用,不是遗漏。
|
|
79
|
+
*/
|
|
80
|
+
export function createCiRenderer(options: CiRendererOptions): FeedbackRenderer {
|
|
81
|
+
const { io } = options;
|
|
82
|
+
|
|
83
|
+
// 距上一次「有意义的输出」(任意一次永久事件)过了多久,用来判断要不要追加一条 heartbeat。
|
|
84
|
+
// "plan" 本身就是第一次永久事件,天然把这个时钟从 0 开始计——不需要像 agent.ts 那样为
|
|
85
|
+
// 「start 行需要真实调度后的 running/queued」而单独等第一次 tick(见文件顶部注释)。
|
|
86
|
+
let lastCheckpointAtMs = 0;
|
|
87
|
+
// "summary" 与 "saved" 是 coordinator.finish() 里连续 emit 的两个独立永久事件(见
|
|
88
|
+
// coordinator.ts 的 finish() 实现,中间不会插入其它事件)——result 收尾需要两者合并,所以
|
|
89
|
+
// "summary" 到达时先记下来,"saved" 到达时才真正写出 result/json/junit/snapshots 几行。
|
|
90
|
+
let pendingSummary: { summary: RunSummary; completion: RunCompletion; reused: number } | undefined;
|
|
91
|
+
|
|
92
|
+
function noteCheckpoint(atMs: number): void {
|
|
93
|
+
lastCheckpointAtMs = atMs;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
appendDurable(event, state) {
|
|
98
|
+
switch (event.type) {
|
|
99
|
+
case "plan": {
|
|
100
|
+
noteCheckpoint(event.at);
|
|
101
|
+
const { shape, reused } = event.plan;
|
|
102
|
+
io.stdout.write(
|
|
103
|
+
`niceeval: start ${kv("total", shape.totalRuns)} ${kv("configs", shape.configs)} ${kv(
|
|
104
|
+
"concurrency",
|
|
105
|
+
shape.maxConcurrency,
|
|
106
|
+
)} ${kv("reused", reused)}\n`,
|
|
107
|
+
);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
case "failure": {
|
|
112
|
+
noteCheckpoint(event.at);
|
|
113
|
+
writeFailureCheckpoint(io, event, state.freshFailureCount);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
case "diagnostic": {
|
|
118
|
+
noteCheckpoint(event.at);
|
|
119
|
+
if (!isFirstOccurrence(state, event.key)) return; // 去重后只追加一次(checklist)
|
|
120
|
+
const parts = [`niceeval: ${event.severity === "error" ? "error" : "warning"}`, kv("key", event.key)];
|
|
121
|
+
if (event.identity?.evalId) parts.push(kv("eval", event.identity.evalId));
|
|
122
|
+
if (event.identity?.experimentId) parts.push(kv("experiment", event.identity.experimentId));
|
|
123
|
+
parts.push(kv("message", event.message));
|
|
124
|
+
io.stdout.write(parts.join(" ") + "\n");
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
case "budget-exhausted": {
|
|
129
|
+
noteCheckpoint(event.at);
|
|
130
|
+
if (!isFirstOccurrence(state, `budget-exhausted:${event.experimentId}`)) return;
|
|
131
|
+
io.stdout.write(
|
|
132
|
+
`niceeval: budget_exhausted ${kv("experiment", event.experimentId)} ${kv(
|
|
133
|
+
"spent",
|
|
134
|
+
event.spent.toFixed(2),
|
|
135
|
+
)} ${kv("unstarted", event.unstarted)}\n`,
|
|
136
|
+
);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
case "interrupted": {
|
|
141
|
+
noteCheckpoint(event.at);
|
|
142
|
+
if (!isFirstOccurrence(state, "interrupted")) return;
|
|
143
|
+
io.stdout.write(`niceeval: interrupted ${kv("elapsed", formatCiSeconds(state.elapsedMs))}\n`);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
case "reporter-error": {
|
|
148
|
+
noteCheckpoint(event.at);
|
|
149
|
+
if (!isFirstOccurrence(state, `reporter-error:${event.reporter}`)) return;
|
|
150
|
+
io.stdout.write(
|
|
151
|
+
`niceeval: reporter_error ${kv("reporter", event.reporter)} ${kv("required", event.required)} ${kv(
|
|
152
|
+
"message",
|
|
153
|
+
event.message,
|
|
154
|
+
)}\n`,
|
|
155
|
+
);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
case "summary":
|
|
160
|
+
// 只记录,不写——见上方 pendingSummary 注释。reused 只在 RunFeedbackState 上,
|
|
161
|
+
// RunSummary 本身没有这个字段,所以在这里(还能读到最新 state)一并存下来。
|
|
162
|
+
pendingSummary = { summary: event.summary, completion: event.completion, reused: state.reused };
|
|
163
|
+
return;
|
|
164
|
+
|
|
165
|
+
case "kept":
|
|
166
|
+
// 人读单行(见 docs/feature/sandbox/cli.md「run 收尾输出」的 ci 形态)。
|
|
167
|
+
io.stderr.write(
|
|
168
|
+
`niceeval: kept sandbox ${event.sandboxId} (${event.provider}) — ${event.identity.evalId} #${event.identity.attempt} ${event.verdict} — enter: niceeval sandbox enter ${event.sandboxId.slice(0, 12)}\n`,
|
|
169
|
+
);
|
|
170
|
+
return;
|
|
171
|
+
|
|
172
|
+
case "saved":
|
|
173
|
+
writeResultBlock(io, pendingSummary, event);
|
|
174
|
+
return;
|
|
175
|
+
|
|
176
|
+
default: {
|
|
177
|
+
// 穷尽性检查:新增 DurableFeedbackEvent 变体时这里编译期报错提醒补上对应分支。
|
|
178
|
+
const exhaustive: never = event;
|
|
179
|
+
return exhaustive;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
},
|
|
183
|
+
|
|
184
|
+
onTick(event, state) {
|
|
185
|
+
const idle = event.at - lastCheckpointAtMs;
|
|
186
|
+
if (idle < CI_HEARTBEAT_IDLE_MS) return;
|
|
187
|
+
noteCheckpoint(event.at);
|
|
188
|
+
io.stdout.write(progressLine(state, event.elapsedMs) + "\n");
|
|
189
|
+
},
|
|
190
|
+
|
|
191
|
+
// 没有 clearDynamic/redrawDynamic/activity/onLifecycle:见文件顶部注释——CI 不维护动态
|
|
192
|
+
// 区域、不逐次输出 provisioning retry/backoff、不逐条展示 active phase 或 passed attempt,
|
|
193
|
+
// 这些由「不实现对应可选钩子」天然满足。
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** 某个去重 key 在 `state.diagnostics` 里是不是第一次出现(count === 1)。budget-exhausted /
|
|
198
|
+
* interrupted / reporter-error 的去重 key 计算方式与 reducer.ts 完全一致(见该文件的
|
|
199
|
+
* `budget-exhausted:${experimentId}` / `"interrupted"` / `reporter-error:${reporter}`),
|
|
200
|
+
* 两处必须同源,不能各写一份——与 agent.ts 同一原则(这几个 key 是 `DurableFeedbackEvent`
|
|
201
|
+
* 自身语义的一部分,不是 reducer 实现细节,三个 renderer 都要各自能算出同一个 key)。 */
|
|
202
|
+
function isFirstOccurrence(state: RunFeedbackState, key: string): boolean {
|
|
203
|
+
return (state.diagnostics.find((d) => d.key === key)?.count ?? 0) <= 1;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// ───────────────────────── 字段转义 / envelope 拼装(纯函数) ─────────────────────────
|
|
207
|
+
|
|
208
|
+
type FieldValue = string | number | boolean;
|
|
209
|
+
|
|
210
|
+
/** 字段值需要空格(或引号)时用 JSON string 转义,不依赖 locale(与 agent.ts 同一规则,独立
|
|
211
|
+
* 实现)。没有空格/引号的值保持裸 token。 */
|
|
212
|
+
function escapeFieldValue(raw: string): string {
|
|
213
|
+
return /[\s"]/.test(raw) ? JSON.stringify(raw) : raw;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function kv(key: string, value: FieldValue): string {
|
|
217
|
+
return typeof value === "string" ? `${key}=${escapeFieldValue(value)}` : `${key}=${value}`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** "0s" / "60s" / "128s" 风格,不做分钟换算——见文件顶部「为什么统一用原始秒数」。 */
|
|
221
|
+
function formatCiSeconds(ms: number): string {
|
|
222
|
+
return `${Math.max(0, Math.floor(ms / 1000))}s`;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function progressLine(state: RunFeedbackState, elapsedMs: number): string {
|
|
226
|
+
return [
|
|
227
|
+
"niceeval: progress",
|
|
228
|
+
kv("elapsed", formatCiSeconds(elapsedMs)),
|
|
229
|
+
kv("reused", state.reused),
|
|
230
|
+
kv("running", state.running),
|
|
231
|
+
kv("queued", state.queued),
|
|
232
|
+
kv("completed", state.completed),
|
|
233
|
+
].join(" ");
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ───────────────────────── failure / error checkpoint(stdout,立即追加,带上限) ─────────────────────────
|
|
237
|
+
|
|
238
|
+
function writeFailureCheckpoint(io: FeedbackIO, event: DurableFeedbackEvent & { type: "failure" }, countSoFar: number): void {
|
|
239
|
+
if (countSoFar === CI_FAILURE_CAP + 1) {
|
|
240
|
+
// 越过上限的第一条给一次 suppressed 提示,不是每条失败都重复一遍——完整清单由 JSON/JUnit
|
|
241
|
+
// 保存(checklist 第四条),这里只保证「知道发生了折叠」。必须在下面的「> CAP 一律静默」
|
|
242
|
+
// 之前判断,否则这个分支永远不可达。
|
|
243
|
+
io.stdout.write(
|
|
244
|
+
`niceeval: warning ${kv("key", "failures-suppressed")} ${kv(
|
|
245
|
+
"message",
|
|
246
|
+
"further failed/errored attempts are not streamed individually; see the JSON/JUnit report",
|
|
247
|
+
)}\n`,
|
|
248
|
+
);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (countSoFar > CI_FAILURE_CAP) return; // 上限之后完全静默
|
|
252
|
+
const word = event.verdict === "errored" ? "errored" : "failed";
|
|
253
|
+
const parts = [`niceeval: ${word}`, kv("locator", event.locator), kv("eval", event.identity.evalId)];
|
|
254
|
+
if (event.identity.experimentId) parts.push(kv("experiment", event.identity.experimentId));
|
|
255
|
+
if (event.phase) parts.push(kv("phase", event.phase));
|
|
256
|
+
if (event.assertion) {
|
|
257
|
+
parts.push(kv("severity", event.assertion.severity), kv("assertion", event.assertion.assertion));
|
|
258
|
+
if (event.assertion.matcher !== undefined) parts.push(kv("matcher", event.assertion.matcher));
|
|
259
|
+
if (event.assertion.expected !== undefined) parts.push(kv("expected", event.assertion.expected));
|
|
260
|
+
if (event.assertion.received !== undefined) parts.push(kv("received", event.assertion.received));
|
|
261
|
+
if (event.assertion.score !== undefined) parts.push(kv("score", event.assertion.score));
|
|
262
|
+
if (event.assertion.threshold !== undefined) parts.push(kv("threshold", event.assertion.threshold));
|
|
263
|
+
if (event.assertion.reason !== undefined) parts.push(kv("reason", event.assertion.reason));
|
|
264
|
+
if (event.assertion.additionalFailures > 0) parts.push(kv("additional_failures", event.assertion.additionalFailures));
|
|
265
|
+
} else {
|
|
266
|
+
parts.push(kv("reason", event.reason));
|
|
267
|
+
}
|
|
268
|
+
io.stdout.write(parts.join(" ") + "\n");
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ───────────────────────── result 收尾(stdout,status/counts/reused/unstarted/duration + json/junit/snapshots) ─────────────────────────
|
|
272
|
+
|
|
273
|
+
/** completion 优先于 verdict 计数——interrupted/incomplete 时即便全部 attempt 都通过,也不能
|
|
274
|
+
* 说 "passed"(checklist 第六条:budget unstarted → incomplete、用户中断 → interrupted)。
|
|
275
|
+
* required reporter 失败同样折进 "failed":它不是 `CompletionStatus` 的第四个值(那个枚举
|
|
276
|
+
* 只有 complete/incomplete/interrupted 三态,见 ../types.ts 的 `CompletionStatus` 注释),
|
|
277
|
+
* 但「要求的 reporter 写失败」必须让退出码非零(见 `computeCiExitCode`),状态词也不能显示
|
|
278
|
+
* 一个会被误读成「全绿」的 "passed"。 */
|
|
279
|
+
function resultStatusWord(summary: RunSummary, completion: RunCompletion): "passed" | "failed" | "incomplete" | "interrupted" {
|
|
280
|
+
if (completion.status === "interrupted") return "interrupted";
|
|
281
|
+
if (completion.status === "incomplete") return "incomplete";
|
|
282
|
+
if (completion.reporterErrors.some((e) => e.required)) return "failed";
|
|
283
|
+
return summary.failed > 0 || summary.errored > 0 ? "failed" : "passed";
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function resultLine(summary: RunSummary, completion: RunCompletion, reused: number): string {
|
|
287
|
+
const parts = [
|
|
288
|
+
`niceeval: result=${resultStatusWord(summary, completion)}`,
|
|
289
|
+
kv("passed", summary.passed),
|
|
290
|
+
kv("failed", summary.failed),
|
|
291
|
+
kv("errored", summary.errored),
|
|
292
|
+
];
|
|
293
|
+
// reused/unstarted 都只在 > 0 时才出现——与 cli.md 的两个字面例子完全对应:正常例子有
|
|
294
|
+
// `reused=18`(无 unstarted),budget 例子有 `unstarted=4`(无 reused)。两个字段互不排斥,
|
|
295
|
+
// 理论上可以同时出现(既有携入结果、又有 budget 未派发),这里的条件判断天然支持这种情况。
|
|
296
|
+
if (reused > 0) parts.push(kv("reused", reused));
|
|
297
|
+
if (completion.unstarted > 0) parts.push(kv("unstarted", completion.unstarted));
|
|
298
|
+
parts.push(kv("duration", formatCiSeconds(summary.durationMs)));
|
|
299
|
+
return parts.join(" ");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** 快照路径的共同「组」段(如 `.niceeval/ci/bub/<snapshot>` → `ci`),用于折叠成
|
|
303
|
+
* `.niceeval/<group>/<N snapshots>`。多个 experiment 组混在一次 invocation 里时组名不唯一,
|
|
304
|
+
* 不猜一个可能错的前缀,退化成不带路径前缀的 `<N snapshots>`——与 human.ts/agent.ts 同名逻辑
|
|
305
|
+
* 刻意分开实现(见文件顶部注释),不是共享同一个函数。 */
|
|
306
|
+
function deriveResultGroup(paths: readonly string[]): string | undefined {
|
|
307
|
+
const groups = new Set<string>();
|
|
308
|
+
for (const p of paths) {
|
|
309
|
+
const seg = p.split("/")[1];
|
|
310
|
+
if (seg) groups.add(seg);
|
|
311
|
+
}
|
|
312
|
+
return groups.size === 1 ? [...groups][0] : undefined;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** 单条路径直接原样打印;多条折叠成 `<N snapshots>`(见 cli.md「CI 怎么用」的字面例子:
|
|
316
|
+
* `.niceeval/ci/<3 snapshots>`)——CI「同一种事件一行」的整体约束(checklist/docs 都强调
|
|
317
|
+
* 一行一事件,方便日志搜索)决定了这里不能像 human/agent 那样逐条缩进列出。 */
|
|
318
|
+
function snapshotsValue(paths: readonly string[]): string {
|
|
319
|
+
if (paths.length === 1) return paths[0]!;
|
|
320
|
+
const group = deriveResultGroup(paths);
|
|
321
|
+
return group ? `.niceeval/${group}/<${paths.length} snapshots>` : `<${paths.length} snapshots>`;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function writeResultBlock(
|
|
325
|
+
io: FeedbackIO,
|
|
326
|
+
pending: { summary: RunSummary; completion: RunCompletion; reused: number } | undefined,
|
|
327
|
+
event: DurableFeedbackEvent & { type: "saved" },
|
|
328
|
+
): void {
|
|
329
|
+
if (!pending) return; // 不应发生:coordinator.finish() 恒先 emit "summary" 再 emit "saved"。
|
|
330
|
+
const { summary, completion, reused } = pending;
|
|
331
|
+
io.stdout.write(resultLine(summary, completion, reused) + "\n");
|
|
332
|
+
// json=/junit=/snapshots= 各自独立一行,且只在真的写出对应产物时才打印(checklist 第五条:
|
|
333
|
+
// 「随后打印实际生成的 JSON/JUnit/快照路径」——"实际生成的",不是"配置里要求生成的")。
|
|
334
|
+
if (event.json) io.stdout.write(`niceeval: json=${event.json}\n`);
|
|
335
|
+
if (event.junit) io.stdout.write(`niceeval: junit=${event.junit}\n`);
|
|
336
|
+
if (event.paths.length > 0) io.stdout.write(`niceeval: snapshots=${snapshotsValue(event.paths)}\n`);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// ───────────────────────── 退出码(CompletionStatus 驱动,checklist 第六条) ─────────────────────────
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* 把 `RunSummary` + `RunCompletion` 折成 CLI 退出码——checklist:「budget unstarted →
|
|
343
|
+
* incomplete + 非零;required reporter 失败 → 非零;用户中断 → 130」。这是这个 stage 承担的
|
|
344
|
+
* 「CompletionStatus 驱动的退出码语义」交付物(见 docs/feature/experiments/cli.md「CI 怎么用」
|
|
345
|
+
* 的退出码表):不看 CLI flag、不读 process.env,是纯函数,后续 CLI 接线阶段(plan section G)
|
|
346
|
+
* 直接调用它算最终 `process.exit()` 的参数,不需要在 `src/cli.ts` 里重新写一遍这套判断
|
|
347
|
+
* (今天 `src/cli.ts` 的 `evalLevelStats`/`failedExit` 只看 failed/errored,不知道
|
|
348
|
+
* incomplete/interrupted/reporter 失败这三类——接线时用这个函数的结果替换那段逻辑,而不是
|
|
349
|
+
* 在旁边再加一层判断)。这个函数不是"仅 --output ci 才用"——退出码是进程级机器信号,与选了
|
|
350
|
+
* 哪个反馈 profile 无关,human/agent 跑同一次 run 得到的退出码必须一致;放在这个模块只是因为
|
|
351
|
+
* CI 是最早、最明确需要 CompletionStatus 三态参与退出码判断的消费者。
|
|
352
|
+
*
|
|
353
|
+
* 不在这里处理「2 = CLI/runner 未捕获崩溃」——那是进程级 uncaught exception/rejection 处理器
|
|
354
|
+
* 的职责,不是「一次 run 正常收尾后该给什么退出码」的问题,不应该由 completion 驱动。
|
|
355
|
+
*/
|
|
356
|
+
export function computeCiExitCode(summary: RunSummary, completion: RunCompletion): number {
|
|
357
|
+
if (completion.status === "interrupted") return 130;
|
|
358
|
+
if (completion.status === "incomplete") return 1;
|
|
359
|
+
if (completion.reporterErrors.some((e) => e.required)) return 1;
|
|
360
|
+
return summary.failed > 0 || summary.errored > 0 ? 1 : 0;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// ───────────────────────── `--dry`(ci profile):稳定预览,不经 coordinator ─────────────────────────
|
|
364
|
+
|
|
365
|
+
/** 一个 (config, eval) 组合在 `--dry` 预览里的一行。与 human.ts 的 `HumanDryPlanRow` 字段形状
|
|
366
|
+
* 相似但刻意分开定义(见文件顶部「不共享」注释)——ci 用 `experimentId ?? who` 拼出单一
|
|
367
|
+
* `experiment=` 字段,不像 human 那样分成 `who` + `experimentSuffix` 两段拼接文案。 */
|
|
368
|
+
export interface CiDryPlanRow {
|
|
369
|
+
experimentId?: string;
|
|
370
|
+
who: string;
|
|
371
|
+
evalIds: readonly string[];
|
|
372
|
+
runs: number;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
export interface CiDryPlanInput {
|
|
376
|
+
/** 总 attempt 数(= Σ 每个 (config, eval) 组合的 evalIds.length × runs)。 */
|
|
377
|
+
total: number;
|
|
378
|
+
/** 去重后候选 eval 数,口径同 `RunFeedbackPlan.shape.evals`。 */
|
|
379
|
+
evals: number;
|
|
380
|
+
configs: number;
|
|
381
|
+
rows: readonly CiDryPlanRow[];
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** dry run 不派发 attempt,没有 `RunFeedbackState` 可言,所以这是独立于 `FeedbackRenderer`/
|
|
385
|
+
* coordinator 的纯函数(与 `computeCiExitCode`、agent.ts 的 `renderAgentPlanEnvelope` 同一定位)。
|
|
386
|
+
* 仍然是固定 ASCII `niceeval: <word> key=value` 行,不读 i18n、不设行数上限——预览的组合数
|
|
387
|
+
* 就是这次调用会展开的完整矩阵,不是运行中可能无界增长的事件流,没有「防止刷屏」的顾虑。 */
|
|
388
|
+
export function renderCiDryPlan(input: CiDryPlanInput): string {
|
|
389
|
+
const lines = [
|
|
390
|
+
`niceeval: plan ${kv("total", input.total)} ${kv("evals", input.evals)} ${kv("configs", input.configs)}`,
|
|
391
|
+
];
|
|
392
|
+
for (const row of input.rows) {
|
|
393
|
+
lines.push(
|
|
394
|
+
`niceeval: plan-row ${kv("experiment", row.experimentId ?? row.who)} ${kv(
|
|
395
|
+
"evals",
|
|
396
|
+
row.evalIds.join(", ") || "(no matches)",
|
|
397
|
+
)} ${kv("runs", row.runs)}`,
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
return lines.join("\n") + "\n";
|
|
401
|
+
}
|