niceeval 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -12
- package/README.zh.md +8 -9
- package/docs-site/.mintignore +13 -0
- package/docs-site/AGENTS.md +46 -0
- package/docs-site/concepts/adapter.mdx +287 -0
- package/docs-site/concepts/assert.mdx +243 -0
- package/docs-site/concepts/drive.mdx +118 -0
- package/docs-site/concepts/evals.mdx +108 -0
- package/docs-site/concepts/experiment.mdx +37 -0
- package/docs-site/concepts/hitl.mdx +83 -0
- package/docs-site/concepts/judge.mdx +129 -0
- package/docs-site/concepts/overview.mdx +98 -0
- package/docs-site/concepts/tier.mdx +42 -0
- package/docs-site/docs-ref/00-index.md +23 -0
- package/docs-site/docs-ref/01-page-types.md +43 -0
- package/docs-site/docs-ref/03-style-rules.md +45 -0
- package/docs-site/docs-ref/04-code-examples.md +37 -0
- package/docs-site/docs-ref/05-dx-failure-paths.md +45 -0
- package/docs-site/docs-ref/06-checklists.md +53 -0
- package/docs-site/docs.json +256 -0
- package/docs-site/example/ai-agent-application.mdx +148 -0
- package/docs-site/example/claude-code-codex-plugin.mdx +162 -0
- package/docs-site/example/claude-code-codex-skill.mdx +147 -0
- package/docs-site/example/showcase.mdx +39 -0
- package/docs-site/example/tier1-ai-sdk-v7.mdx +136 -0
- package/docs-site/example/tier1-claude-sdk.mdx +119 -0
- package/docs-site/example/tier1-codex-sdk.mdx +111 -0
- package/docs-site/example/tier1-langgraph.mdx +119 -0
- package/docs-site/example/tier1-pi-sdk.mdx +119 -0
- package/docs-site/favicon.svg +5 -0
- package/docs-site/github-diff.css +135 -0
- package/docs-site/github-diff.js +11 -0
- package/docs-site/guides/authoring.mdx +129 -0
- package/docs-site/guides/ci-integration.mdx +97 -0
- package/docs-site/guides/connect-otel.mdx +209 -0
- package/docs-site/guides/connect-your-agent.mdx +221 -0
- package/docs-site/guides/dataset-fanout.mdx +98 -0
- package/docs-site/guides/experiments.mdx +71 -0
- package/docs-site/guides/fixtures.mdx +147 -0
- package/docs-site/guides/reporters.mdx +113 -0
- package/docs-site/guides/runner.mdx +79 -0
- package/docs-site/guides/sandbox-agent.mdx +138 -0
- package/docs-site/guides/sandbox-backends.mdx +64 -0
- package/docs-site/guides/scoring-guide.mdx +92 -0
- package/docs-site/guides/viewing-results.mdx +195 -0
- package/docs-site/guides/write-experiment.mdx +81 -0
- package/docs-site/guides/write-send.mdx +347 -0
- package/docs-site/images/adapter-tiers-zh.svg +60 -0
- package/docs-site/images/agent-turn-roundtrip-en.svg +62 -0
- package/docs-site/images/agent-turn-roundtrip-zh.svg +77 -0
- package/docs-site/images/hitl-handshake-zh.svg +89 -0
- package/docs-site/images/logo.svg +6 -0
- package/docs-site/index.mdx +181 -0
- package/docs-site/introduction.mdx +141 -0
- package/docs-site/quickstart.mdx +136 -0
- package/docs-site/reference/builtin-agents.mdx +161 -0
- package/docs-site/reference/capabilities.mdx +76 -0
- package/docs-site/reference/cli.mdx +120 -0
- package/docs-site/reference/define-agent.mdx +168 -0
- package/docs-site/reference/define-config.mdx +41 -0
- package/docs-site/reference/define-eval.mdx +160 -0
- package/docs-site/reference/events.mdx +131 -0
- package/docs-site/reference/expect.mdx +112 -0
- package/docs-site/tracker.js +8 -0
- package/docs-site/zh/concepts/adapter.mdx +286 -0
- package/docs-site/zh/concepts/assert.mdx +243 -0
- package/docs-site/zh/concepts/drive.mdx +118 -0
- package/docs-site/zh/concepts/evals.mdx +124 -0
- package/docs-site/zh/concepts/experiment.mdx +37 -0
- package/docs-site/zh/concepts/hitl.mdx +83 -0
- package/docs-site/zh/concepts/judge.mdx +130 -0
- package/docs-site/zh/concepts/overview.mdx +109 -0
- package/docs-site/zh/concepts/tier.mdx +44 -0
- package/docs-site/zh/example/ai-agent-application.mdx +152 -0
- package/docs-site/zh/example/claude-code-codex-plugin.mdx +163 -0
- package/docs-site/zh/example/claude-code-codex-skill.mdx +147 -0
- package/docs-site/zh/example/showcase.mdx +39 -0
- package/docs-site/zh/example/tier1-ai-sdk-v7.mdx +140 -0
- package/docs-site/zh/example/tier1-claude-sdk.mdx +122 -0
- package/docs-site/zh/example/tier1-codex-sdk.mdx +117 -0
- package/docs-site/zh/example/tier1-langgraph.mdx +123 -0
- package/docs-site/zh/example/tier1-pi-sdk.mdx +122 -0
- package/docs-site/zh/guides/agent-feedback-loop.mdx +79 -0
- package/docs-site/zh/guides/authoring.mdx +129 -0
- package/docs-site/zh/guides/ci-integration.mdx +97 -0
- package/docs-site/zh/guides/connect-otel.mdx +208 -0
- package/docs-site/zh/guides/connect-your-agent.mdx +226 -0
- package/docs-site/zh/guides/dataset-fanout.mdx +83 -0
- package/docs-site/zh/guides/experiments.mdx +72 -0
- package/docs-site/zh/guides/fixtures.mdx +147 -0
- package/docs-site/zh/guides/official-adapters.mdx +132 -0
- package/docs-site/zh/guides/reporters.mdx +113 -0
- package/docs-site/zh/guides/runner.mdx +82 -0
- package/docs-site/zh/guides/sandbox-agent.mdx +139 -0
- package/docs-site/zh/guides/sandbox-backends.mdx +75 -0
- package/docs-site/zh/guides/scoring-guide.mdx +92 -0
- package/docs-site/zh/guides/viewing-results.mdx +195 -0
- package/docs-site/zh/guides/write-experiment.mdx +101 -0
- package/docs-site/zh/guides/write-send.mdx +353 -0
- package/docs-site/zh/index.mdx +180 -0
- package/docs-site/zh/introduction.mdx +140 -0
- package/docs-site/zh/quickstart.mdx +137 -0
- package/docs-site/zh/reference/builtin-agents.mdx +364 -0
- package/docs-site/zh/reference/capabilities.mdx +76 -0
- package/docs-site/zh/reference/cli.mdx +140 -0
- package/docs-site/zh/reference/define-agent.mdx +461 -0
- package/docs-site/zh/reference/define-config.mdx +107 -0
- package/docs-site/zh/reference/define-eval.mdx +674 -0
- package/docs-site/zh/reference/events.mdx +252 -0
- package/docs-site/zh/reference/expect.mdx +208 -0
- package/package.json +3 -2
- package/src/agents/bub.ts +15 -5
- package/src/agents/claude-code.ts +8 -5
- package/src/agents/codex.ts +9 -5
- package/src/agents/sdk-streams.test.ts +4 -4
- package/src/agents/sdk-streams.ts +14 -9
- package/src/agents/types.ts +40 -1
- package/src/agents/ui-message-stream.ts +1 -0
- package/src/cli.ts +36 -7
- package/src/context/types.ts +140 -0
- package/src/expect/index.ts +15 -8
- package/src/i18n/en.ts +19 -6
- package/src/i18n/zh-CN.ts +19 -6
- package/src/o11y/parsers/bub.test.ts +71 -0
- package/src/o11y/parsers/bub.ts +5 -0
- package/src/o11y/types.ts +27 -2
- package/src/runner/reporters/artifacts.ts +11 -3
- package/src/runner/reporters/live.ts +45 -8
- package/src/runner/run.ts +71 -21
- package/src/runner/types.ts +45 -2
- package/src/sandbox/types.ts +18 -0
- package/src/scoring/types.ts +5 -0
- package/src/shared/types.ts +3 -0
- package/src/util.test.ts +26 -1
- package/src/util.ts +16 -0
- package/src/view/app/App.tsx +2 -2
- package/src/view/app/components/AttemptModal.tsx +2 -0
- package/src/view/app/components/CopyControls.tsx +87 -28
- package/src/view/app/i18n.ts +3 -3
- package/src/view/client-dist/app.css +1 -1
- package/src/view/client-dist/app.js +18 -18
- package/docs/README.md +0 -120
- package/docs/adapters/README.md +0 -60
- package/docs/adapters/authoring.md +0 -179
- package/docs/adapters/coding-agent-skills-plugins.md +0 -324
- package/docs/adapters/collection.md +0 -128
- package/docs/adapters/contract.md +0 -264
- package/docs/adapters/reference/agent-eval.md +0 -215
- package/docs/adapters/reference/agent-loop-apis.md +0 -69
- package/docs/adapters/reference/claude-code-otel-telemetry.md +0 -100
- package/docs/adapters/reference/eve-protocol.md +0 -127
- package/docs/adapters/reference/otel-genai.md +0 -107
- package/docs/adapters/reference/otel-instrumentation.md +0 -65
- package/docs/adapters/targets.md +0 -99
- package/docs/architecture.md +0 -140
- package/docs/assertions.md +0 -387
- package/docs/capabilities-by-construction.md +0 -48
- package/docs/cli.md +0 -173
- package/docs/concepts.md +0 -106
- package/docs/e2e-ci.md +0 -332
- package/docs/eval-authoring.md +0 -319
- package/docs/experiments.md +0 -157
- package/docs/getting-started.md +0 -224
- package/docs/multi-agent.md +0 -145
- package/docs/observability.md +0 -337
- package/docs/origin-integration.md +0 -195
- package/docs/references.md +0 -35
- package/docs/reports.md +0 -551
- package/docs/results-format.md +0 -228
- package/docs/results-lib.md +0 -191
- package/docs/runner.md +0 -104
- package/docs/sandbox.md +0 -276
- package/docs/scoring.md +0 -119
- package/docs/source-map.md +0 -126
- package/docs/tier-sync.md +0 -193
- package/docs/view.md +0 -194
- package/docs/vision.md +0 -88
package/src/runner/types.ts
CHANGED
|
@@ -97,7 +97,8 @@ export interface RunShape {
|
|
|
97
97
|
configs: number;
|
|
98
98
|
/** 总 attempt 数(evals × configs × runs);逐行输出与汇总计数都按它。 */
|
|
99
99
|
totalRuns: number;
|
|
100
|
-
/**
|
|
100
|
+
/** 本次运行实际生效的全局并发数(flag/env/config/sandbox 默认值解析后的结果);
|
|
101
|
+
* 实验级 maxConcurrency 只在该实验内部限流,不改这个全局值。 */
|
|
101
102
|
maxConcurrency: number;
|
|
102
103
|
}
|
|
103
104
|
|
|
@@ -122,12 +123,17 @@ export type ReporterEvent =
|
|
|
122
123
|
export interface EvalDef {
|
|
123
124
|
/** 路径推导,定义里禁止手写。 */
|
|
124
125
|
id?: string;
|
|
126
|
+
/** 一句话描述,展示在 `niceeval list` 和 view 里;纯说明,不影响调度或打分。 */
|
|
125
127
|
description?: string;
|
|
126
|
-
|
|
128
|
+
/** 标签,供 CLI `--tag` 过滤和 view 分类;与 id 前缀过滤是两套独立的筛选维度。 */
|
|
127
129
|
tags?: string[];
|
|
130
|
+
/** 覆盖项目级 Config.judge,只对这一个 eval 生效(如换个更贵的评审模型)。 */
|
|
128
131
|
judge?: JudgeConfig;
|
|
132
|
+
/** 覆盖 / 追加项目级 Config.reporters,只对这一个 eval 生效。 */
|
|
129
133
|
reporters?: Reporter[];
|
|
134
|
+
/** 覆盖项目级 / CLI 的单次 attempt 超时(毫秒),只对这一个 eval 生效。 */
|
|
130
135
|
timeoutMs?: number;
|
|
136
|
+
/** 任意附加元数据,原样透传进 EvalResult,不参与调度或打分;供自定义 reporter 消费。 */
|
|
131
137
|
metadata?: Record<string, unknown>;
|
|
132
138
|
/**
|
|
133
139
|
* eval 级预置:拿到沙箱(已上传 workspace + git 基线 + 装好依赖前)。
|
|
@@ -135,6 +141,7 @@ export interface EvalDef {
|
|
|
135
141
|
* (如 `runCommand("apt-get", ["install", …], { root: true })`),跨后端语义一致。
|
|
136
142
|
*/
|
|
137
143
|
setup?: (sandbox: Sandbox) => Promise<void | Cleanup> | void | Cleanup;
|
|
144
|
+
/** eval 主体:拿到 TestContext,驱动对话 / 沙箱操作并就地断言。 */
|
|
138
145
|
test(t: TestContext): Promise<void> | void;
|
|
139
146
|
}
|
|
140
147
|
|
|
@@ -148,20 +155,43 @@ export interface DiscoveredEval extends EvalDef {
|
|
|
148
155
|
}
|
|
149
156
|
|
|
150
157
|
export interface ExperimentDef {
|
|
158
|
+
/** 路径推导,定义里禁止手写(defineExperiment 会拒绝显式传入)。 */
|
|
151
159
|
id?: string;
|
|
160
|
+
/** 一句话描述,展示在 view / CLI 里;纯说明,不影响调度或打分。 */
|
|
152
161
|
description?: string;
|
|
162
|
+
/**
|
|
163
|
+
* 必填:这个实验跑哪个 agent(defineSandboxAgent / defineAgent 的产物)。运行配置的
|
|
164
|
+
* agent 归属完全由这里决定——EvalDef.agent 不参与(见其字段注释)。
|
|
165
|
+
*/
|
|
153
166
|
agent: Agent;
|
|
154
167
|
/** 单个模型(agent 留空时实验决定);省略=用 agent 原生默认。跨模型对比写多个实验文件,别用数组。 */
|
|
155
168
|
model?: string;
|
|
156
169
|
/** 模型推理努力程度(如 "low"/"medium"/"high",取值由具体模型/adapter 决定);省略=用 agent 原生默认。经 ctx.reasoningEffort 透给 adapter 与 eval。 */
|
|
157
170
|
reasoningEffort?: string;
|
|
171
|
+
/** 传给每次 attempt 的 flags,经 t.flags 暴露给 eval;与 CLI flag 合并(CLI 优先)。 */
|
|
158
172
|
flags?: Record<string, unknown>;
|
|
173
|
+
/** 同一 eval 重复跑几次(结果各计一条 attempt);省略/CLI `--runs` 覆盖时默认 1。 */
|
|
159
174
|
runs?: number;
|
|
175
|
+
/** 一次重复(runs > 1)里某次 attempt 失败后是否跳过剩余重复;省略默认 true(提前退出省钱)。 */
|
|
160
176
|
earlyExit?: boolean;
|
|
177
|
+
/** 这个实验覆盖哪些 eval:"*" 全部、字符串数组按 id 前缀、或自定义谓词;省略等价于 "*"。 */
|
|
161
178
|
evals?: "*" | string[] | ((id: string) => boolean);
|
|
179
|
+
/** 覆盖项目级 / CLI 的单次 attempt 超时(毫秒),只对这个实验生效。 */
|
|
162
180
|
timeoutMs?: number;
|
|
181
|
+
/** 覆盖项目级 Config.sandbox,只对这个实验生效。 */
|
|
163
182
|
sandbox?: SandboxOption;
|
|
183
|
+
/**
|
|
184
|
+
* 本实验的花费上限(USD)。调度器按「已花 + 在飞预估」的护栏口径逼近上限时限流,
|
|
185
|
+
* 累计花费到顶后跳过这个实验剩下未起飞的 attempt 并上报一次 `run:budgetExceeded`
|
|
186
|
+
* (已在飞的 attempt 仍会跑完)。
|
|
187
|
+
*/
|
|
164
188
|
budget?: number;
|
|
189
|
+
/**
|
|
190
|
+
* 本实验自己的并发上限:调度器只对这个实验的 attempt 限流,同批其它实验不受影响,
|
|
191
|
+
* 仍按全局并发(CLI / env / config / 沙箱默认)跑。用于串行化有共享状态的实验
|
|
192
|
+
* (如跨 eval 累积记忆:`maxConcurrency: 1` 保证 attempt 按 eval 顺序一个个跑),
|
|
193
|
+
* 或给撞后端限额的实验单独降速。
|
|
194
|
+
*/
|
|
165
195
|
maxConcurrency?: number;
|
|
166
196
|
}
|
|
167
197
|
|
|
@@ -176,11 +206,17 @@ export interface Config {
|
|
|
176
206
|
* 可传字符串,或按 locale 提供多语言(如 `{ en: "...", "zh-CN": "..." }`),随 view 语言切换。
|
|
177
207
|
*/
|
|
178
208
|
name?: LocalizedText;
|
|
209
|
+
/** 项目级默认沙箱后端(docker / vercel / e2b / custom);experiment / CLI flag 可覆盖。 */
|
|
179
210
|
sandbox?: SandboxOption;
|
|
211
|
+
/** 上传进沙箱的工作区根目录,省略则用项目根;eval 的 sandbox 视图从这里起步。 */
|
|
180
212
|
workspace?: string;
|
|
213
|
+
/** 项目级默认 judge 配置(model / baseUrl / apiKeyEnv);EvalDef.judge 可按 eval 覆盖。 */
|
|
181
214
|
judge?: JudgeConfig;
|
|
215
|
+
/** 项目级默认 reporter 列表(如落盘 / 上传结果);EvalDef.reporters 会与它合并。 */
|
|
182
216
|
reporters?: Reporter[];
|
|
217
|
+
/** 项目级默认并发上限;CLI flag / env / experiment 的同名设置优先级更高。 */
|
|
183
218
|
maxConcurrency?: number;
|
|
219
|
+
/** 项目级默认单次 attempt 超时(毫秒);CLI flag / experiment / EvalDef 的同名设置优先级更高。 */
|
|
184
220
|
timeoutMs?: number;
|
|
185
221
|
/**
|
|
186
222
|
* OTLP 接收配置,niceeval 项目内唯一入口(不读 NICEEVAL_OTLP_* 环境变量)。
|
|
@@ -203,9 +239,13 @@ export interface Config {
|
|
|
203
239
|
|
|
204
240
|
/** 每百万 token 的美元单价;省略的桶退回 `inputPerMTok`(cache token 本质也是 input)。 */
|
|
205
241
|
export interface PriceOverride {
|
|
242
|
+
/** 普通输入 token 单价。 */
|
|
206
243
|
inputPerMTok: number;
|
|
244
|
+
/** 输出 token 单价。 */
|
|
207
245
|
outputPerMTok: number;
|
|
246
|
+
/** cache 命中(读)token 单价,省略则退回 inputPerMTok。 */
|
|
208
247
|
cacheReadPerMTok?: number;
|
|
248
|
+
/** cache 写入 token 单价,省略则退回 inputPerMTok。 */
|
|
209
249
|
cacheWritePerMTok?: number;
|
|
210
250
|
}
|
|
211
251
|
|
|
@@ -235,6 +275,9 @@ export interface AgentRun {
|
|
|
235
275
|
evalFilter: (id: string) => boolean;
|
|
236
276
|
experimentId?: string;
|
|
237
277
|
strict?: boolean;
|
|
278
|
+
/** 本配置自己的并发上限(来自 ExperimentDef.maxConcurrency):调度器为它单建信号量,
|
|
279
|
+
* attempt 先过这道闸再占全局并发位;省略则只受全局并发约束。 */
|
|
280
|
+
maxConcurrency?: number;
|
|
238
281
|
}
|
|
239
282
|
|
|
240
283
|
export interface RunOptions {
|
package/src/sandbox/types.ts
CHANGED
|
@@ -93,7 +93,9 @@ export type SandboxSpec = DockerSandboxSpec | VercelSandboxSpec | E2BSandboxSpec
|
|
|
93
93
|
export type SandboxOption = SandboxSpec;
|
|
94
94
|
|
|
95
95
|
export interface CommandOptions {
|
|
96
|
+
/** 追加/覆盖本命令的环境变量(与沙箱默认环境叠加,不清空默认值;各后端会保留自己固定的 `PATH` 等变量,不保证能被这里覆盖)。 */
|
|
96
97
|
env?: Record<string, string>;
|
|
98
|
+
/** 本命令的工作目录;省略时落到 `Sandbox.workdir`。相对路径按 workdir 解析,绝对路径原样使用。 */
|
|
97
99
|
cwd?: string;
|
|
98
100
|
/**
|
|
99
101
|
* 把本命令的输出也送进沙箱的「原生日志流」(于是 `docker logs` / Docker UI 的 Logs
|
|
@@ -115,20 +117,36 @@ export interface CommandOptions {
|
|
|
115
117
|
}
|
|
116
118
|
|
|
117
119
|
export interface Sandbox {
|
|
120
|
+
/** 沙箱内项目/工作区根目录的绝对路径(agent 命令的默认 cwd,也是 git baseline 提交的位置)。各方法的相对路径都以此为基准解析,省略 `cwd`/`targetDir` 时也落到这里。 */
|
|
118
121
|
readonly workdir: string;
|
|
122
|
+
/**
|
|
123
|
+
* 执行单个命令,`args` 作为独立 argv 传递、不经 shell 解释(无 `&&`、管道、通配符展开)。
|
|
124
|
+
* 只想跑一个可执行文件、参数来自外部输入、担心注入时优先用它。
|
|
125
|
+
*/
|
|
119
126
|
runCommand(cmd: string, args?: string[], opts?: CommandOptions): Promise<CommandResult>;
|
|
127
|
+
/**
|
|
128
|
+
* 执行一整段脚本,经 shell(bash)解释,支持 `&&`、管道、`$()`、重定向等。
|
|
129
|
+
* 需要拼多条命令或做条件判断时用它。
|
|
130
|
+
*/
|
|
120
131
|
runShell(script: string, opts?: CommandOptions): Promise<CommandResult>;
|
|
132
|
+
/** 读取沙箱内文件的文本内容(UTF-8)。文件不存在时抛错,不返回空字符串——需要容错请自行 `.catch()`。 */
|
|
121
133
|
readFile(path: string): Promise<string>;
|
|
134
|
+
/** 检查沙箱内路径是否存在。跨后端语义不完全一致:仅保证对普通文件可靠,对目录路径的行为不同后端不保证一致。 */
|
|
122
135
|
fileExists(path: string): Promise<boolean>;
|
|
123
136
|
/**
|
|
124
137
|
* 一次 shell 往返读全部源码文件(按扩展名收、按目录/文件名忽略)。
|
|
125
138
|
* 取代每个 eval 目录里手写的 find + 逐文件 readFile。
|
|
126
139
|
*/
|
|
127
140
|
readSourceFiles(opts?: ReadSourceFilesOptions): Promise<SourceFiles>;
|
|
141
|
+
/** 写入若干文本文件(内容已在内存里的字符串);是 `uploadFiles` 的文本特化,省略 `targetDir` 落到 workdir。 */
|
|
128
142
|
writeFiles(files: Record<string, string>, targetDir?: string): Promise<void>;
|
|
143
|
+
/** 批量写入若干文件,内容可以是文本或二进制 Buffer;省略 `targetDir` 落到 workdir。 */
|
|
129
144
|
uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void>;
|
|
145
|
+
/** 把本地磁盘上的一个目录整体上传进沙箱(递归读取本地文件后按 `uploadFiles` 写入);`opts.ignore` 是排除规则,省略 `targetDir` 落到 workdir。 */
|
|
130
146
|
uploadDirectory(localDir: string, targetDir?: string, opts?: { ignore?: string[] }): Promise<void>;
|
|
147
|
+
/** 销毁沙箱占用的计算资源(容器/microVM)。调用后沙箱不可再用;是否可安全重复调用因后端而异,不要依赖这一点。 */
|
|
131
148
|
stop(): Promise<void>;
|
|
149
|
+
/** 本沙箱的稳定标识(各后端原生 ID,如 Docker 容器 ID 前缀);用于跨调用关联同一沙箱的会话状态,也用于日志展示。 */
|
|
132
150
|
readonly sandboxId: string;
|
|
133
151
|
/**
|
|
134
152
|
* 本地 OTLP 接收器的目标 host。
|
package/src/scoring/types.ts
CHANGED
|
@@ -9,7 +9,12 @@ export interface ValueAssertion {
|
|
|
9
9
|
readonly severity: Severity;
|
|
10
10
|
readonly threshold?: number;
|
|
11
11
|
score(value: unknown): number | Promise<number>;
|
|
12
|
+
/** 转成硬门槛断言:未达阈值(省略 threshold 则按 score > 0 判定)整条 eval 判为 failed。返回新实例,不改原对象。 */
|
|
12
13
|
gate(threshold?: number): ValueAssertion;
|
|
14
|
+
/**
|
|
15
|
+
* 转成软阈值断言:未达 threshold 时该条记为 failed,但默认不拖累整条 eval 的 outcome;
|
|
16
|
+
* `--strict` 运行下,软阈值失败也会把整条 eval 的 outcome 计为 failed。返回新实例,不改原对象。
|
|
17
|
+
*/
|
|
13
18
|
atLeast(threshold: number): ValueAssertion;
|
|
14
19
|
}
|
|
15
20
|
|
package/src/shared/types.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// 各域的类型住在各自目录的 types.ts(o11y / sandbox / agents / scoring / context / runner),
|
|
3
3
|
// src/types.ts 是聚合 facade —— 模块代码统一从那里 import,不必记住每个类型的家。
|
|
4
4
|
|
|
5
|
+
/** JSON 可表达的任意值(递归定义),用于事件流 / 工具输入输出等跨进程/跨语言传递的数据。 */
|
|
5
6
|
export type JsonValue =
|
|
6
7
|
| string
|
|
7
8
|
| number
|
|
@@ -10,6 +11,7 @@ export type JsonValue =
|
|
|
10
11
|
| JsonValue[]
|
|
11
12
|
| { [key: string]: JsonValue };
|
|
12
13
|
|
|
14
|
+
/** 断言的严重级:"gate" 失败必判整轮 failed;"soft" 默认只记录不拦截,仅在 `--strict` 模式或显式设阈值未达标时才计入失败。 */
|
|
13
15
|
export type Severity = "gate" | "soft";
|
|
14
16
|
|
|
15
17
|
/**
|
|
@@ -28,6 +30,7 @@ export interface SourceArtifact {
|
|
|
28
30
|
content: string;
|
|
29
31
|
}
|
|
30
32
|
|
|
33
|
+
/** 通用清理闭包(setup 返回值 / teardown 的形状),异步同步皆可,统一在 finally 里执行。 */
|
|
31
34
|
export type Cleanup = () => Promise<void> | void;
|
|
32
35
|
|
|
33
36
|
/**
|
package/src/util.test.ts
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { formatThrown } from "./util.ts";
|
|
2
|
+
import { formatThrown, upsertManagedBlock } from "./util.ts";
|
|
3
|
+
|
|
4
|
+
describe("upsertManagedBlock", () => {
|
|
5
|
+
const begin = "<!-- BEGIN:x -->";
|
|
6
|
+
const end = "<!-- END:x -->";
|
|
7
|
+
|
|
8
|
+
it("appends the block to existing content, separated by a blank line", () => {
|
|
9
|
+
const out = upsertManagedBlock("# My project\n", begin, end, "rules");
|
|
10
|
+
expect(out).toBe(`# My project\n\n${begin}\nrules\n${end}\n`);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("creates just the block when the file is empty", () => {
|
|
14
|
+
expect(upsertManagedBlock("", begin, end, "rules")).toBe(`${begin}\nrules\n${end}\n`);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it("replaces only the content between existing markers, preserving what surrounds them", () => {
|
|
18
|
+
const before = `above\n\n${begin}\nold rules\n${end}\n\nbelow\n`;
|
|
19
|
+
const out = upsertManagedBlock(before, begin, end, "new rules");
|
|
20
|
+
expect(out).toBe(`above\n\n${begin}\nnew rules\n${end}\n\nbelow\n`);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it("is idempotent for the same content", () => {
|
|
24
|
+
const once = upsertManagedBlock("# hi\n", begin, end, "rules");
|
|
25
|
+
expect(upsertManagedBlock(once, begin, end, "rules")).toBe(once);
|
|
26
|
+
});
|
|
27
|
+
});
|
|
3
28
|
|
|
4
29
|
describe("formatThrown", () => {
|
|
5
30
|
it("uses the stack trace when available, so the report can locate the throw site", () => {
|
package/src/util.ts
CHANGED
|
@@ -37,6 +37,22 @@ export function pad4(n: number): string {
|
|
|
37
37
|
return String(n).padStart(4, "0");
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* 在文本里维护一个带 BEGIN/END 标记的托管区块(AGENTS.md 的 niceeval 区块用)。
|
|
42
|
+
* 标记已存在 → 只替换两个标记之间的内容(升级时刷新指引,区块外的用户内容不动);
|
|
43
|
+
* 不存在 → 追加到末尾(与已有内容之间空一行)。
|
|
44
|
+
*/
|
|
45
|
+
export function upsertManagedBlock(source: string, begin: string, end: string, content: string): string {
|
|
46
|
+
const block = `${begin}\n${content}\n${end}`;
|
|
47
|
+
const beginIdx = source.indexOf(begin);
|
|
48
|
+
const endIdx = source.indexOf(end);
|
|
49
|
+
if (beginIdx !== -1 && endIdx !== -1 && endIdx > beginIdx) {
|
|
50
|
+
return source.slice(0, beginIdx) + block + source.slice(endIdx + end.length);
|
|
51
|
+
}
|
|
52
|
+
if (source.trim() === "") return `${block}\n`;
|
|
53
|
+
return `${source.replace(/\n*$/, "")}\n\n${block}\n`;
|
|
54
|
+
}
|
|
55
|
+
|
|
40
56
|
/** 把任意值安全地转成简短字符串(报告 / 日志用)。 */
|
|
41
57
|
export function brief(value: unknown, max = 200): string {
|
|
42
58
|
let s: string;
|
package/src/view/app/App.tsx
CHANGED
|
@@ -9,7 +9,7 @@ import { Metric } from "./components/primitives.tsx";
|
|
|
9
9
|
import { GroupSelector } from "./components/GroupSelector.tsx";
|
|
10
10
|
import { CostScoreChart } from "./components/CostScoreChart.tsx";
|
|
11
11
|
import { ExperimentTable } from "./components/ExperimentTable.tsx";
|
|
12
|
-
import {
|
|
12
|
+
import { CopyFixPrompt } from "./components/CopyControls.tsx";
|
|
13
13
|
import { AttemptModal } from "./components/AttemptModal.tsx";
|
|
14
14
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs.tsx";
|
|
15
15
|
import { RunsView } from "./pages/RunsPage.tsx";
|
|
@@ -239,7 +239,7 @@ export function App({ data }: { data: ViewData }) {
|
|
|
239
239
|
value={query}
|
|
240
240
|
onChange={(e) => setQuery(e.target.value)}
|
|
241
241
|
/>
|
|
242
|
-
<
|
|
242
|
+
<CopyFixPrompt rows={filtered} t={t} />
|
|
243
243
|
</div>
|
|
244
244
|
</div>
|
|
245
245
|
{rows.length ? (
|
|
@@ -5,6 +5,7 @@ import { artifactUrl } from "../lib/artifact-url.ts";
|
|
|
5
5
|
import { asEvents, asSources } from "../lib/guards.ts";
|
|
6
6
|
import { outcomeClass, outcomeLabel } from "../lib/outcome.ts";
|
|
7
7
|
import { CodeView, NoSourceBody } from "./CodeView.tsx";
|
|
8
|
+
import { CopyAttemptPrompt } from "./CopyControls.tsx";
|
|
8
9
|
import { LazyArtifact } from "./LazyArtifact.tsx";
|
|
9
10
|
import { Dialog, DialogClose, DialogContent, DialogTitle } from "./ui/dialog.tsx";
|
|
10
11
|
import { Badge } from "./ui/badge.tsx";
|
|
@@ -44,6 +45,7 @@ export function AttemptModal({ result, onClose, t }: { result: ViewResult; onClo
|
|
|
44
45
|
</DialogTitle>
|
|
45
46
|
{result.description ? <span className="truncate text-xs text-muted">{result.description}</span> : null}
|
|
46
47
|
</div>
|
|
48
|
+
<CopyAttemptPrompt result={result} t={t} />
|
|
47
49
|
<DialogClose
|
|
48
50
|
aria-label={t("action.close")}
|
|
49
51
|
className="grid h-7 w-7 shrink-0 place-items-center rounded-md border border-transparent text-sm text-muted transition-colors hover:border-line hover:bg-panel-2 hover:text-text"
|
|
@@ -23,41 +23,100 @@ export function CopyReason({ text, t }: { text: string; t: T }) {
|
|
|
23
23
|
);
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
/** 修复 prompt 的一条失败条目;路径均相对 view 输入根(默认 `.niceeval/`)。 */
|
|
27
|
+
export interface FixPromptEntry {
|
|
28
|
+
experiment: string;
|
|
29
|
+
evalId: string;
|
|
30
|
+
outcome: string;
|
|
31
|
+
reason: string;
|
|
32
|
+
artifactBase?: string;
|
|
33
|
+
summaryPath?: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function toFixPromptEntry(r: ViewResult, experimentLabel: string): FixPromptEntry {
|
|
37
|
+
return {
|
|
38
|
+
experiment: r.experimentId ?? experimentLabel,
|
|
39
|
+
evalId: r.id,
|
|
40
|
+
outcome: r.outcome,
|
|
41
|
+
reason: reasonFor(r, failingAssertions(r)),
|
|
42
|
+
artifactBase: r.artifactBase,
|
|
43
|
+
summaryPath: r.attemptRef ? `${r.attemptRef.run}/summary.json` : undefined,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* 学 Next.js 16.3 的「Copy prompt」:把失败打包成一段可直接粘给 coding agent 的修复
|
|
49
|
+
* prompt——失败清单 + 工件路径 + 先读随包文档 / 判断缺陷在哪一侧 / 重跑验证的步骤。
|
|
50
|
+
* prompt 面向 agent,固定英文;按钮文案走界面 i18n。
|
|
51
|
+
*/
|
|
52
|
+
export function buildFixPrompt(entries: FixPromptEntry[]): string {
|
|
53
|
+
const failures = entries
|
|
54
|
+
.map((e, i) =>
|
|
55
|
+
[
|
|
56
|
+
`${i + 1}. eval "${e.evalId}" [experiment ${e.experiment}] — ${e.outcome}`,
|
|
57
|
+
e.reason ? ` reason: ${e.reason}` : null,
|
|
58
|
+
e.artifactBase ? ` artifacts: ${e.artifactBase}/` : null,
|
|
59
|
+
e.summaryPath ? ` summary: ${e.summaryPath}` : null,
|
|
60
|
+
]
|
|
61
|
+
.filter(Boolean)
|
|
62
|
+
.join("\n"),
|
|
63
|
+
)
|
|
64
|
+
.join("\n");
|
|
65
|
+
const experiments = [...new Set(entries.map((e) => e.experiment))].join(" / ");
|
|
66
|
+
return [
|
|
67
|
+
"Fix the failing evals from this niceeval run.",
|
|
68
|
+
"",
|
|
69
|
+
"## Failures",
|
|
70
|
+
failures,
|
|
71
|
+
"",
|
|
72
|
+
"## Steps",
|
|
73
|
+
"1. niceeval is NOT in your training data. Read the relevant guide in `node_modules/niceeval/docs-site/` (English at the top level, Chinese under `zh/`) before changing anything.",
|
|
74
|
+
"2. The paths above are relative to the results directory (default `.niceeval/`). For each failure, read `events.json` in its artifacts directory — the full agent transcript including tool calls — plus `trace.json` (execution trace) and `diff.json` (workspace diff) when present, to see what actually happened.",
|
|
75
|
+
"3. Decide which side the defect is on: the program under test, or the eval itself (over-tight assertion, wrong fixture, missing setup). Fix that side; do not weaken assertions just to turn the run green.",
|
|
76
|
+
`4. Re-run: \`npx niceeval exp ${experiments || "<experiment>"} <eval-id-prefix>\`. Already-passing evals are skipped by the fingerprint cache; pass \`--force\` to re-run everything.`,
|
|
77
|
+
"5. Read the new `summary.json` path the CLI prints and confirm these failures are gone.",
|
|
78
|
+
].join("\n");
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function CopyFixPrompt({ rows, t }: { rows: ViewRow[]; t: T }) {
|
|
27
82
|
const [copied, setCopied] = useState(false);
|
|
28
83
|
|
|
29
|
-
const
|
|
84
|
+
const entries = rows.flatMap((row: ViewRow) =>
|
|
30
85
|
(row.results ?? [])
|
|
31
|
-
.filter((r: ViewResult) =>
|
|
32
|
-
|
|
33
|
-
})
|
|
34
|
-
.map((r: ViewResult) => {
|
|
35
|
-
const failedAssertions = failingAssertions(r);
|
|
36
|
-
const reason = reasonFor(r, failedAssertions);
|
|
37
|
-
// 静态 HTML 里只有相对 view 根的工件路径;宿主机绝对路径不再进 viewData。
|
|
38
|
-
const traceBase = r.artifactBase;
|
|
39
|
-
const tracePath = r.hasTrace && traceBase ? `${traceBase}/trace.json` : null;
|
|
40
|
-
return { experimentName: row.label, evalId: r.id, reason, tracePath };
|
|
41
|
-
})
|
|
86
|
+
.filter((r: ViewResult) => r.outcome === "failed" || r.outcome === "errored")
|
|
87
|
+
.map((r: ViewResult) => toFixPromptEntry(r, row.label)),
|
|
42
88
|
);
|
|
43
89
|
|
|
44
|
-
if (!
|
|
90
|
+
if (!entries.length) return null;
|
|
45
91
|
|
|
46
92
|
const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
47
93
|
event.stopPropagation();
|
|
48
|
-
const text = errorEntries
|
|
49
|
-
.map(({ experimentName, evalId, reason, tracePath }: { experimentName: string; evalId: string; reason: string; tracePath: string | null }) =>
|
|
50
|
-
[
|
|
51
|
-
`实验: ${experimentName} Eval: ${evalId}`,
|
|
52
|
-
reason ? `错误: ${reason}` : null,
|
|
53
|
-
tracePath ? `Trace: ${tracePath}` : null,
|
|
54
|
-
]
|
|
55
|
-
.filter(Boolean)
|
|
56
|
-
.join("\n")
|
|
57
|
-
)
|
|
58
|
-
.join("\n\n");
|
|
59
94
|
try {
|
|
60
|
-
await copyText(
|
|
95
|
+
await copyText(buildFixPrompt(entries));
|
|
96
|
+
setCopied(true);
|
|
97
|
+
setTimeout(() => setCopied(false), 1500);
|
|
98
|
+
} catch {
|
|
99
|
+
setCopied(false);
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
return (
|
|
104
|
+
<button className={`copy-all-errors${copied ? " is-copied" : ""}`} onClick={copy} title={t("action.copyPrompt")}>
|
|
105
|
+
{copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
|
|
106
|
+
<span>{copied ? t("action.copied") : `${t("action.copyPrompt")} (${entries.length})`}</span>
|
|
107
|
+
</button>
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** attempt 弹窗里的单条版:只打包当前 attempt 的失败,供逐条转交 agent。 */
|
|
112
|
+
export function CopyAttemptPrompt({ result, t }: { result: ViewResult; t: T }) {
|
|
113
|
+
const [copied, setCopied] = useState(false);
|
|
114
|
+
if (result.outcome !== "failed" && result.outcome !== "errored") return null;
|
|
115
|
+
|
|
116
|
+
const copy = async (event: React.MouseEvent<HTMLButtonElement>) => {
|
|
117
|
+
event.stopPropagation();
|
|
118
|
+
try {
|
|
119
|
+
await copyText(buildFixPrompt([toFixPromptEntry(result, result.agent)]));
|
|
61
120
|
setCopied(true);
|
|
62
121
|
setTimeout(() => setCopied(false), 1500);
|
|
63
122
|
} catch {
|
|
@@ -66,9 +125,9 @@ export function CopyAllErrors({ rows, t }: { rows: ViewRow[]; t: T }) {
|
|
|
66
125
|
};
|
|
67
126
|
|
|
68
127
|
return (
|
|
69
|
-
<button className={`copy-all-errors${copied ? " is-copied" : ""}`} onClick={copy} title={t("action.
|
|
128
|
+
<button className={`copy-all-errors${copied ? " is-copied" : ""}`} onClick={copy} title={t("action.copyPrompt")}>
|
|
70
129
|
{copied ? <Check aria-hidden="true" /> : <Copy aria-hidden="true" />}
|
|
71
|
-
<span>{copied ? t("action.copied") :
|
|
130
|
+
<span>{copied ? t("action.copied") : t("action.copyPrompt")}</span>
|
|
72
131
|
</button>
|
|
73
132
|
);
|
|
74
133
|
}
|
package/src/view/app/i18n.ts
CHANGED
|
@@ -67,7 +67,7 @@ export type MessageKey =
|
|
|
67
67
|
| "status.skipped"
|
|
68
68
|
| "action.close"
|
|
69
69
|
| "action.copyReason"
|
|
70
|
-
| "action.
|
|
70
|
+
| "action.copyPrompt"
|
|
71
71
|
| "action.copied"
|
|
72
72
|
| "trace.loading"
|
|
73
73
|
| "trace.loadFailed"
|
|
@@ -181,7 +181,7 @@ const dictionaries: Record<Locale, Dictionary> = {
|
|
|
181
181
|
"status.skipped": "skipped",
|
|
182
182
|
"action.close": "Close",
|
|
183
183
|
"action.copyReason": "Copy reason",
|
|
184
|
-
"action.
|
|
184
|
+
"action.copyPrompt": "Copy fix prompt",
|
|
185
185
|
"action.copied": "Copied",
|
|
186
186
|
"trace.loading": "loading...",
|
|
187
187
|
"trace.loadFailed": "load failed (static report has no server - use niceeval view):",
|
|
@@ -292,7 +292,7 @@ const dictionaries: Record<Locale, Dictionary> = {
|
|
|
292
292
|
"status.skipped": "跳过",
|
|
293
293
|
"action.close": "关闭",
|
|
294
294
|
"action.copyReason": "复制原因",
|
|
295
|
-
"action.
|
|
295
|
+
"action.copyPrompt": "复制修复 Prompt",
|
|
296
296
|
"action.copied": "已复制",
|
|
297
297
|
"trace.loading": "加载中...",
|
|
298
298
|
"trace.loadFailed": "加载失败(静态报告没有服务端 - 请用 niceeval view):",
|