niceeval 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +9 -9
  2. package/README.zh.md +14 -14
  3. package/docs/README.md +119 -0
  4. package/docs/adapters/README.md +60 -0
  5. package/docs/adapters/authoring.md +179 -0
  6. package/docs/adapters/coding-agent-skills-plugins.md +324 -0
  7. package/docs/adapters/collection.md +128 -0
  8. package/docs/adapters/contract.md +263 -0
  9. package/docs/adapters/reference/agent-eval.md +215 -0
  10. package/docs/adapters/reference/agent-loop-apis.md +69 -0
  11. package/docs/adapters/reference/claude-code-otel-telemetry.md +100 -0
  12. package/docs/adapters/reference/eve-protocol.md +127 -0
  13. package/docs/adapters/reference/otel-genai.md +107 -0
  14. package/docs/adapters/reference/otel-instrumentation.md +65 -0
  15. package/docs/adapters/targets.md +99 -0
  16. package/docs/architecture.md +140 -0
  17. package/docs/assertions.md +388 -0
  18. package/docs/capabilities-by-construction.md +48 -0
  19. package/docs/cli.md +171 -0
  20. package/docs/concepts.md +106 -0
  21. package/docs/e2e-ci.md +295 -0
  22. package/docs/eval-authoring.md +319 -0
  23. package/docs/experiments.md +154 -0
  24. package/docs/getting-started.md +224 -0
  25. package/docs/multi-agent.md +145 -0
  26. package/docs/observability.md +333 -0
  27. package/docs/origin-integration.md +195 -0
  28. package/docs/references.md +35 -0
  29. package/docs/results-format.md +228 -0
  30. package/docs/runner.md +104 -0
  31. package/docs/sandbox.md +276 -0
  32. package/docs/scoring.md +119 -0
  33. package/docs/source-map.md +106 -0
  34. package/docs/tier-sync.md +177 -0
  35. package/docs/view.md +157 -0
  36. package/docs/vision.md +88 -0
  37. package/package.json +46 -17
  38. package/src/agents/ai-sdk-otel.ts +0 -0
  39. package/src/agents/ai-sdk.test.ts +377 -0
  40. package/src/agents/ai-sdk.ts +577 -0
  41. package/src/agents/bub.ts +30 -37
  42. package/src/agents/claude-code.ts +7 -4
  43. package/src/agents/codex.ts +15 -10
  44. package/src/agents/index.ts +43 -1
  45. package/src/agents/sdk-streams.test.ts +145 -0
  46. package/src/agents/sdk-streams.ts +457 -0
  47. package/src/agents/shared.ts +77 -39
  48. package/src/agents/streaming.test.ts +152 -0
  49. package/src/agents/streaming.ts +195 -0
  50. package/src/agents/types.ts +218 -0
  51. package/src/agents/ui-message-stream.test.ts +249 -0
  52. package/src/agents/ui-message-stream.ts +372 -0
  53. package/src/cli.ts +124 -77
  54. package/src/context/context.test.ts +203 -7
  55. package/src/context/context.ts +158 -49
  56. package/src/context/session.test.ts +96 -0
  57. package/src/context/session.ts +119 -9
  58. package/src/context/types.ts +205 -0
  59. package/src/define.ts +4 -14
  60. package/src/expect/index.ts +8 -71
  61. package/src/i18n/core.ts +28 -0
  62. package/src/i18n/en.ts +47 -5
  63. package/src/i18n/index.ts +5 -17
  64. package/src/i18n/zh-CN.ts +47 -5
  65. package/src/index.ts +12 -0
  66. package/src/loaders/index.ts +8 -39
  67. package/src/o11y/otlp/canonical.ts +7 -6
  68. package/src/o11y/otlp/mappers/codex.ts +10 -8
  69. package/src/o11y/otlp/mappers/index.ts +9 -21
  70. package/src/o11y/otlp/receiver.ts +25 -7
  71. package/src/o11y/otlp/sandbox-receiver.ts +57 -21
  72. package/src/o11y/otlp/turn-otel.test.ts +110 -0
  73. package/src/o11y/otlp/turn-otel.ts +125 -0
  74. package/src/o11y/parsers/bub.ts +13 -11
  75. package/src/o11y/parsers/claude-code.ts +27 -51
  76. package/src/o11y/parsers/codex.ts +29 -46
  77. package/src/o11y/parsers/index.ts +5 -14
  78. package/src/o11y/tool-names.test.ts +61 -0
  79. package/src/o11y/tool-names.ts +74 -0
  80. package/src/o11y/types.ts +135 -0
  81. package/src/runner/attempt.ts +456 -0
  82. package/src/runner/fingerprint.ts +59 -0
  83. package/src/runner/remote-sandbox.ts +53 -0
  84. package/src/runner/report.ts +53 -0
  85. package/src/runner/reporters/artifacts.ts +25 -4
  86. package/src/runner/reporters/console.ts +2 -8
  87. package/src/runner/reporters/live.ts +2 -8
  88. package/src/runner/reporters/shared.ts +15 -0
  89. package/src/runner/reporters/table.ts +10 -62
  90. package/src/runner/run.ts +114 -619
  91. package/src/runner/types.ts +237 -0
  92. package/src/sandbox/checkpoint.ts +15 -13
  93. package/src/sandbox/docker-stream.ts +87 -0
  94. package/src/sandbox/docker.ts +42 -120
  95. package/src/sandbox/e2b.ts +24 -75
  96. package/src/sandbox/local-files.ts +33 -0
  97. package/src/sandbox/paths.test.ts +85 -0
  98. package/src/sandbox/paths.ts +45 -0
  99. package/src/sandbox/resolve.ts +10 -34
  100. package/src/sandbox/shell.ts +17 -0
  101. package/src/sandbox/source-files.ts +42 -2
  102. package/src/sandbox/types.ts +158 -0
  103. package/src/sandbox/vercel.ts +55 -71
  104. package/src/scoring/collector.ts +3 -2
  105. package/src/scoring/judge.ts +72 -146
  106. package/src/scoring/match.ts +79 -0
  107. package/src/scoring/types.ts +76 -0
  108. package/src/shared/aggregate.ts +48 -0
  109. package/src/shared/format.ts +20 -0
  110. package/src/shared/outcome.ts +48 -0
  111. package/src/shared/types.ts +37 -0
  112. package/src/types.ts +20 -911
  113. package/src/view/aggregate.ts +103 -0
  114. package/src/view/app/App.tsx +26 -5
  115. package/src/view/app/components/AttemptModal.tsx +12 -3
  116. package/src/view/app/components/CodeView.tsx +9 -5
  117. package/src/view/app/components/CopyControls.tsx +4 -4
  118. package/src/view/app/components/ExperimentTable.tsx +5 -5
  119. package/src/view/app/i18n.ts +31 -7
  120. package/src/view/app/lib/format.ts +17 -20
  121. package/src/view/app/lib/outcome.ts +4 -16
  122. package/src/view/app/main.tsx +3 -5
  123. package/src/view/app/pages/RunsPage.tsx +2 -2
  124. package/src/view/app/pages/TracesPage.tsx +2 -2
  125. package/src/view/app/shared.ts +2 -1
  126. package/src/view/app/types.ts +6 -43
  127. package/src/view/client-dist/app.css +1 -1
  128. package/src/view/client-dist/app.js +18 -18
  129. package/src/view/index.ts +24 -401
  130. package/src/view/loader.ts +234 -0
  131. package/src/view/server.ts +136 -0
  132. package/src/view/shared/types.ts +64 -0
  133. package/src/view/styles.css +60 -9
@@ -0,0 +1,237 @@
1
+ // runner 域类型:结果 / 汇总 / reporter 契约,eval / experiment / config 定义,
2
+ // 以及调度器的编排类型(AgentRun / RunOptions / Attempt)。
3
+
4
+ import type { Cleanup, LocalizedText, SourceArtifact } from "../shared/types.ts";
5
+ import type { O11ySummary, StreamEvent, TraceSpan, Usage } from "../o11y/types.ts";
6
+ import type { Agent } from "../agents/types.ts";
7
+ import type { Sandbox, SandboxOption } from "../sandbox/types.ts";
8
+ import type { AssertionResult, DiffData, JudgeConfig, ResultOutcome } from "../scoring/types.ts";
9
+ import type { TestContext } from "../context/types.ts";
10
+
11
+ // ───────────────────────── 结果 / 报告 ─────────────────────────
12
+
13
+ export interface ExperimentRunInfo {
14
+ id?: string;
15
+ flags?: Record<string, unknown>;
16
+ runs?: number;
17
+ earlyExit?: boolean;
18
+ sandbox?: string;
19
+ timeoutMs?: number;
20
+ budget?: number;
21
+ }
22
+
23
+ export interface EvalResult {
24
+ id: string;
25
+ description?: string;
26
+ experimentId?: string;
27
+ experiment?: ExperimentRunInfo;
28
+ agent: string;
29
+ model?: string;
30
+ outcome: ResultOutcome;
31
+ fingerprint?: string;
32
+ attempt: number;
33
+ /** 本 attempt 开始的墙钟时刻(ISO);view 按 eval 粒度展示「何时跑的」。 */
34
+ startedAt?: string;
35
+ durationMs: number;
36
+ assertions: AssertionResult[];
37
+ usage?: Usage;
38
+ estimatedCostUSD?: number;
39
+ error?: string;
40
+ skipReason?: string;
41
+ events?: StreamEvent[];
42
+ /** test 引用到的 eval 源码(按 loc 收集),供 view 渲染 github-diff 式代码视图。 */
43
+ sources?: SourceArtifact[];
44
+ o11y?: O11ySummary;
45
+ /** agent 经 OpenTelemetry 导出的运行追踪(有 tracing 能力且收到 span 时)。 */
46
+ trace?: TraceSpan[];
47
+ diff?: DiffData;
48
+ rawTranscript?: string;
49
+ // ── 拆分工件的引用(Artifacts 报告器写 summary.json 时填;view 按需懒加载)──
50
+ /** 本 attempt 工件目录(相对 run 根),下有 events/trace/o11y/diff.json。 */
51
+ artifactsDir?: string;
52
+ /** view 拼好的工件目录(相对 view 输入根,供前端 fetch);loadSummaries 注入。 */
53
+ artifactBase?: string;
54
+ /** 工件目录的绝对路径;loadSummaries 注入,供复制/展示用。 */
55
+ artifactAbsBase?: string;
56
+ hasTrace?: boolean;
57
+ hasEvents?: boolean;
58
+ hasSources?: boolean;
59
+ }
60
+
61
+ /** `summary.json` 的格式标记;把 niceeval 报告和其它工具的同名文件区分开。 */
62
+ export const RESULTS_FORMAT = "niceeval.results";
63
+ /** 结果格式版本,只在破坏兼容读取时递增;读取器只认相同版本,缺失按 1。见 docs/results-format.md。 */
64
+ export const RESULTS_SCHEMA_VERSION = 1;
65
+
66
+ export interface RunSummary {
67
+ /** 恒为 "niceeval.results";和 schemaVersion、producer 一起构成持久化契约,永不移动或改名。 */
68
+ format?: typeof RESULTS_FORMAT;
69
+ /** 结果格式版本;与读取器不同即视为不兼容,提示用 producer.version 对应的 niceeval 查看。 */
70
+ schemaVersion?: number;
71
+ /** 写这份报告的 niceeval;version 唯一用途是拼 `npx niceeval@<version> view` 提示。 */
72
+ producer?: { name: "niceeval"; version?: string; commit?: string };
73
+ /** 项目名(来自 config.name),透传给 `niceeval view` 顶部 hero 显示。 */
74
+ name?: LocalizedText;
75
+ agent: string;
76
+ model?: string;
77
+ startedAt: string;
78
+ completedAt: string;
79
+ passed: number;
80
+ /** 断言不通过的数量;不包含 errored。 */
81
+ failed: number;
82
+ skipped: number;
83
+ /** 环境、超时、adapter、agent runtime 等执行错误数量;与 failed 互斥。 */
84
+ errored: number;
85
+ durationMs: number;
86
+ usage?: Usage;
87
+ estimatedCostUSD?: number;
88
+ results: EvalResult[];
89
+ outputDir?: string;
90
+ }
91
+
92
+ /** onRunStart 的运行规模:去重后 eval 数 × 配置(agent×model×flags)数 → 总运行(attempt)数。 */
93
+ export interface RunShape {
94
+ /** 去重后实际要跑的 eval 数(= evals.length)。 */
95
+ evals: number;
96
+ /** (agent, model, flags) 配置组合数;compare 多 agent 时 > 1。 */
97
+ configs: number;
98
+ /** 总 attempt 数(evals × configs × runs);逐行输出与汇总计数都按它。 */
99
+ totalRuns: number;
100
+ }
101
+
102
+ export interface Reporter {
103
+ onEvent?(event: ReporterEvent): void | Promise<void>;
104
+ onRunStart?(evals: { id: string }[], agent: Agent, shape?: RunShape): void | Promise<void>;
105
+ onEvalComplete?(result: EvalResult): void | Promise<void>;
106
+ onRunComplete?(summary: RunSummary): void | Promise<void>;
107
+ }
108
+
109
+ export type ReporterEvent =
110
+ | { type: "run:start"; evals: { id: string }[]; agent: Agent; shape: RunShape }
111
+ | { type: "eval:start"; eval: { id: string }; agent: Agent; attempt: number; experimentId?: string }
112
+ | { type: "eval:complete"; result: EvalResult }
113
+ | { type: "run:earlyExit"; evalId: string; experimentId?: string }
114
+ | { type: "run:budgetExceeded"; budget: number; spent: number }
115
+ | { type: "run:saved"; summary: RunSummary }
116
+ | { type: "run:summary"; summary: RunSummary };
117
+
118
+ // ───────────────────────── eval / experiment / config 定义 ─────────────────────────
119
+
120
+ export interface EvalDef {
121
+ /** 路径推导,定义里禁止手写。 */
122
+ id?: string;
123
+ description?: string;
124
+ agent?: string;
125
+ tags?: string[];
126
+ judge?: JudgeConfig;
127
+ reporters?: Reporter[];
128
+ timeoutMs?: number;
129
+ metadata?: Record<string, unknown>;
130
+ /**
131
+ * eval 级预置:拿到沙箱(已上传 workspace + git 基线 + 装好依赖前)。
132
+ * 默认命令以非 root 跑(agent 的自然环境);装系统依赖时给 `runCommand` 传 `{ root: true }`
133
+ * (如 `runCommand("apt-get", ["install", …], { root: true })`),跨后端语义一致。
134
+ */
135
+ setup?: (sandbox: Sandbox) => Promise<void | Cleanup> | void | Cleanup;
136
+ test(t: TestContext): Promise<void> | void;
137
+ }
138
+
139
+ /** 内部:发现后带上 id 的 eval。 */
140
+ export interface DiscoveredEval extends EvalDef {
141
+ id: string;
142
+ /** 定义文件所在目录(解析相对 workspace 用)。 */
143
+ baseDir: string;
144
+ /** 定义文件绝对路径,用于内容指纹缓存。 */
145
+ sourcePath: string;
146
+ }
147
+
148
+ export interface ExperimentDef {
149
+ id?: string;
150
+ description?: string;
151
+ agent: Agent;
152
+ /** 单个模型(agent 留空时实验决定);省略=用 agent 原生默认。跨模型对比写多个实验文件,别用数组。 */
153
+ model?: string;
154
+ flags?: Record<string, unknown>;
155
+ runs?: number;
156
+ earlyExit?: boolean;
157
+ evals?: "*" | string[] | ((id: string) => boolean);
158
+ timeoutMs?: number;
159
+ sandbox?: SandboxOption;
160
+ budget?: number;
161
+ maxConcurrency?: number;
162
+ }
163
+
164
+ export interface DiscoveredExperiment extends ExperimentDef {
165
+ id: string;
166
+ group: string;
167
+ }
168
+
169
+ export interface Config {
170
+ /**
171
+ * 项目名,显示在 `niceeval view` 顶部 hero(`<h1>`),省略则回退到通用标题。
172
+ * 可传字符串,或按 locale 提供多语言(如 `{ en: "...", "zh-CN": "..." }`),随 view 语言切换。
173
+ */
174
+ name?: LocalizedText;
175
+ sandbox?: SandboxOption;
176
+ workspace?: string;
177
+ judge?: JudgeConfig;
178
+ reporters?: Reporter[];
179
+ maxConcurrency?: number;
180
+ timeoutMs?: number;
181
+ /**
182
+ * OTLP 接收配置,niceeval 项目内唯一入口(不读 NICEEVAL_OTLP_* 环境变量)。
183
+ * `port` 钉住接收端口(固定端口模式:长驻服务把 OTEL_EXPORTER_OTLP_ENDPOINT 一次性指到
184
+ * http://localhost:<port>/v1/traces,跑多少次 eval 都不用改)。省略 = 每次运行动态分配
185
+ * 临时端口(经 ctx.telemetry 交给 adapter)。代价:固定端口下同机同时只能跑一个 niceeval 进程,
186
+ * 且该端口被别的进程占用时会报错——换一个空闲端口写回这里即可。
187
+ * `host` 是报给 adapter 的接收端 hostname(而非监听地址,监听地址恒为 0.0.0.0):默认
188
+ * "127.0.0.1";docker 沙箱型 tracing 需要 "host.docker.internal" 之类的场景,或配了隧道
189
+ * 的远程接入,在这里覆盖。
190
+ */
191
+ telemetry?: { host?: string; port?: number };
192
+ }
193
+
194
+ // ───────────────────────── 调度编排 ─────────────────────────
195
+
196
+ /** 一个 (agent, model, flags) 的运行配置 —— 由 CLI / 实验展开。 */
197
+ export interface AgentRun {
198
+ agent: Agent;
199
+ model?: string;
200
+ flags: Record<string, unknown>;
201
+ runs: number;
202
+ earlyExit: boolean;
203
+ sandbox?: SandboxOption;
204
+ timeoutMs?: number;
205
+ budget?: number;
206
+ evalFilter: (id: string) => boolean;
207
+ experimentId?: string;
208
+ strict?: boolean;
209
+ }
210
+
211
+ export interface RunOptions {
212
+ config: Config;
213
+ evals: DiscoveredEval[];
214
+ agentRuns: AgentRun[];
215
+ reporters: Reporter[];
216
+ maxConcurrency: number;
217
+ signal?: AbortSignal;
218
+ /** TTY live display 的进度回调;设置后 attempt 的 log 消息路由到它而不是 stderr。 */
219
+ onProgress?: (evalId: string, who: string, msg: string) => void;
220
+ /** 上次运行的结果。outcome === "passed" 的 (experimentId, evalId) 组合跳过重跑,结果直接合入本次汇总。 */
221
+ priorResults?: EvalResult[];
222
+ /**
223
+ * 非沙箱 tracing agent 的 run 级共享 OTLP 接收池(runEvals 创建并回收;
224
+ * 每个 agent 一个 receiver,attempt 之间共享 —— 被测应用是长驻进程,端点不能随 attempt 换)。
225
+ */
226
+ otelPool?: import("../o11y/otlp/turn-otel.ts").OtelReceiverPool;
227
+ }
228
+
229
+ /** 调度器内部的一次尝试:eval × run × 第几轮。 */
230
+ export interface Attempt {
231
+ evalDef: DiscoveredEval;
232
+ run: AgentRun;
233
+ attempt: number;
234
+ /** agent+model+evalId,用于早停。 */
235
+ key: string;
236
+ fingerprint: string;
237
+ }
@@ -4,36 +4,38 @@
4
4
  // uploadFile — 向沙箱写任意路径的原始字节 ← Buffer
5
5
  //
6
6
  // 原理:
7
- // capture: tar czf /tmp/__fe_cp__.tar.gz <paths> → downloadFile → Buffer
8
- // restore: uploadFile → /tmp/__fe_rs__.tar.gz → tar xzf -C /
7
+ // capture: tar czf /tmp/__fe_cp_<uuid>.tar.gz <paths> → downloadFile → Buffer
8
+ // restore: uploadFile → /tmp/__fe_rs_<uuid>.tar.gz → tar xzf -C /
9
9
  //
10
10
  // tar / binary file I/O 在所有 Linux sandbox(Docker、Vercel、e2b、Modal…)里都支持,
11
11
  // 这段代码对任意 backend 的 Sandbox 实现无改动即可使用。
12
12
 
13
+ import { randomUUID } from "node:crypto";
13
14
  import type { Sandbox } from "../types.ts";
15
+ import { shellQuote } from "./shell.ts";
14
16
  import { t } from "../i18n/index.ts";
15
17
 
16
- const CP_TMP = "/tmp/__fe_cp__.tar.gz";
17
- const RS_TMP = "/tmp/__fe_rs__.tar.gz";
18
+ // 临时 tar 名带随机后缀:同一沙箱被复用 / 并发做 checkpoint 时,固定名会互相覆盖。
19
+ function tmpTarPath(tag: string): string {
20
+ return `/tmp/__fe_${tag}_${randomUUID()}.tar.gz`;
21
+ }
18
22
 
19
23
  /** 把 paths 列出的目录打成 gzip tar,返回 Buffer。 */
20
24
  export async function createCheckpoint(sb: Sandbox, paths: string[]): Promise<Buffer> {
25
+ const tmp = tmpTarPath("cp");
21
26
  const quoted = paths.map(shellQuote).join(" ");
22
27
  // --ignore-failed-read:跳过不存在的路径(如 .cache/uv 可能未建);true 保证 exit 0。
23
- await sb.runShell(`tar czf ${CP_TMP} --ignore-failed-read ${quoted} 2>/dev/null; true`);
24
- const buf = await sb.downloadFile(CP_TMP);
25
- await sb.runShell(`rm -f ${CP_TMP}`);
28
+ await sb.runShell(`tar czf ${tmp} --ignore-failed-read ${quoted} 2>/dev/null; true`);
29
+ const buf = await sb.downloadFile(tmp);
30
+ await sb.runShell(`rm -f ${tmp}`);
26
31
  if (!buf || buf.length === 0) throw new Error(t("checkpoint.emptyTar", { paths: paths.join(", ") }));
27
32
  return buf;
28
33
  }
29
34
 
30
35
  /** 把 createCheckpoint 返回的 Buffer 还原到沙箱根目录。 */
31
36
  export async function restoreCheckpoint(sb: Sandbox, data: Buffer): Promise<void> {
32
- await sb.uploadFile(RS_TMP, data);
37
+ const tmp = tmpTarPath("rs");
38
+ await sb.uploadFile(tmp, data);
33
39
  // -C / 解压到根目录,覆盖同路径文件;rm 用 ; 保证无论成败都清理临时文件。
34
- await sb.runShell(`tar xzf ${RS_TMP} -C / 2>/dev/null; rm -f ${RS_TMP}`);
35
- }
36
-
37
- function shellQuote(s: string): string {
38
- return `'${s.replace(/'/g, "'\\''")}'`;
40
+ await sb.runShell(`tar xzf ${tmp} -C / 2>/dev/null; rm -f ${tmp}`);
39
41
  }
@@ -0,0 +1,87 @@
1
+ // Docker exec 流的 demux 与 tar 打包/解包 helper——从 docker.ts 拆出的纯工具层,
2
+ // 不含任何容器编排逻辑,便于独立测试与复用。
3
+
4
+ import * as tar from "tar-stream";
5
+
6
+ /**
7
+ * Docker exec 复用流(8 字节头 + 载荷)的增量解析器。
8
+ * 头:[stream_type(1B), 0, 0, 0, size(4B 大端)];stream_type:1=stdout,2=stderr。
9
+ *
10
+ * 关键:一帧可能被 Node 的可读流切到【多个 data 事件】里(尤其大输出,如 cat 一个
11
+ * ~100KB 的文件),帧头 / 载荷都可能跨 chunk。所以必须跨 data 累积一个 leftover,
12
+ * 只消费「已到齐的完整帧」,残帧留到下个 data —— 否则会在 chunk 边界丢字节 / 串帧,
13
+ * 表现为 transcript 里随机损坏的行(曾导致 bub tape 的 tool_result/tool_call 被吞)。
14
+ */
15
+ export function createExecDemuxer(): {
16
+ push(chunk: Buffer): void;
17
+ stdout(): string;
18
+ stderr(): string;
19
+ } {
20
+ const stdoutChunks: Buffer[] = [];
21
+ const stderrChunks: Buffer[] = [];
22
+ let buffer: Buffer = Buffer.alloc(0); // 跨 data 累积的残帧;注解为 Buffer 以容纳 concat 的 ArrayBufferLike
23
+
24
+ return {
25
+ push(chunk: Buffer): void {
26
+ buffer = buffer.length ? Buffer.concat([buffer, chunk]) : chunk;
27
+ while (buffer.length >= 8) {
28
+ const streamType = buffer[0];
29
+ const size = buffer.readUInt32BE(4);
30
+ if (buffer.length < 8 + size) break; // 载荷未到齐 → 等下个 chunk
31
+ const payload = buffer.subarray(8, 8 + size);
32
+ if (streamType === 2) stderrChunks.push(payload);
33
+ else stdoutChunks.push(payload); // 1 / 0 / 未知 → 归 stdout
34
+ buffer = buffer.subarray(8 + size);
35
+ }
36
+ },
37
+ stdout(): string {
38
+ return Buffer.concat(stdoutChunks).toString("utf-8");
39
+ },
40
+ stderr(): string {
41
+ return Buffer.concat(stderrChunks).toString("utf-8");
42
+ },
43
+ };
44
+ }
45
+
46
+ /** Readable stream → Buffer。 */
47
+ export async function readableToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
48
+ return new Promise((resolve, reject) => {
49
+ const chunks: Buffer[] = [];
50
+ stream.on("data", (chunk: Buffer | string) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
51
+ stream.on("end", () => resolve(Buffer.concat(chunks)));
52
+ stream.on("error", reject);
53
+ });
54
+ }
55
+
56
+ /** 从单文件 tar 包里提取第一个 entry 的内容。 */
57
+ export async function extractFileFromTar(tarBuf: Buffer): Promise<Buffer> {
58
+ return new Promise((resolve, reject) => {
59
+ const extract = tar.extract();
60
+ let found = false;
61
+ extract.on("entry", (header, stream, next) => {
62
+ if (!found) {
63
+ found = true;
64
+ const chunks: Buffer[] = [];
65
+ stream.on("data", (c: Buffer) => chunks.push(c));
66
+ stream.on("end", () => { resolve(Buffer.concat(chunks)); next(); });
67
+ stream.on("error", reject);
68
+ } else {
69
+ stream.resume();
70
+ next();
71
+ }
72
+ });
73
+ extract.on("finish", () => { if (!found) reject(new Error("tar: no entries found")); });
74
+ extract.on("error", reject);
75
+ extract.end(tarBuf);
76
+ });
77
+ }
78
+
79
+ /** 把若干文件打成 tar 流(putArchive 用)。 */
80
+ export function packFilesToTar(entries: readonly { name: string; content: Buffer }[]): tar.Pack {
81
+ const pack = tar.pack();
82
+ for (const entry of entries) {
83
+ pack.entry({ name: entry.name }, entry.content);
84
+ }
85
+ pack.finalize();
86
+ return pack;
87
+ }
@@ -2,11 +2,8 @@
2
2
  // 改编自 agent-eval 的 docker-sandbox.ts,签名对齐 ../types.ts 的 Sandbox 契约
3
3
  //(runShell/runCommand 的 opts 一律是选项对象,不再用位置参数)。
4
4
 
5
- import { readdir, readFile, stat } from "node:fs/promises";
6
- import { basename, dirname, join, relative, sep } from "node:path";
7
- import { Readable } from "node:stream";
5
+ import { basename, dirname } from "node:path";
8
6
  import Docker from "dockerode";
9
- import * as tar from "tar-stream";
10
7
  import type {
11
8
  Sandbox,
12
9
  CommandResult,
@@ -17,11 +14,17 @@ import type {
17
14
  ReadSourceFilesOptions,
18
15
  } from "../types.ts";
19
16
  import { makeSourceFiles } from "./source-files.ts";
17
+ import {
18
+ DEFAULT_IGNORE_DIRS,
19
+ DEFAULT_IGNORE_FILES,
20
+ DEFAULT_SOURCE_EXTENSIONS,
21
+ collectLocalFiles,
22
+ } from "./local-files.ts";
23
+ import { buildFindScript, shellQuote } from "./shell.ts";
24
+ import { createExecDemuxer, extractFileFromTar, packFilesToTar, readableToBuffer } from "./docker-stream.ts";
25
+ import { resolveSandboxPath } from "./paths.ts";
20
26
  import { t } from "../i18n/index.ts";
21
27
 
22
- const DEFAULT_SOURCE_EXTENSIONS = ["ts", "tsx", "js", "jsx"];
23
- const DEFAULT_IGNORE_DIRS = [".git", ".next", "node_modules", "dist", "build", "coverage"];
24
- const DEFAULT_IGNORE_FILES = ["EVAL.ts", "PROMPT.md"];
25
28
  // 行首哨兵:源码文件几乎不可能出现这一串,用来切分单次 shell 输出里的多份文件。
26
29
  const SOURCE_FILE_MARKER = "::FE-SRC-7b3f9c::";
27
30
 
@@ -47,11 +50,6 @@ const CONTAINER_WORKDIR = "/home/sandbox/workspace";
47
50
  // 容器「主日志」文件:PID1 tail 它 → `docker logs` 实时显示;agent 命令的 stream 输出 tee 进来。
48
51
  const CONTAINER_LOG = "/tmp/niceeval-agent.log";
49
52
 
50
- /** 单引号包裹 + 转义,把一个参数安全嵌进 shell 命令串。 */
51
- function shellQuote(s: string): string {
52
- return `'${s.replace(/'/g, "'\\''")}'`;
53
- }
54
-
55
53
  // 命令默认以非 root 的 node 用户跑:安全 + 兼容(如 Claude Code 在 root 下拒绝
56
54
  // --dangerously-skip-permissions)。node:*-slim 镜像自带 UID/GID 1000 的 node 用户。
57
55
  // 需要 root 的命令(setup 装系统依赖)走 runCommand 的 `{ root: true }`;此默认非 root + 按需提
@@ -67,39 +65,6 @@ const NPM_GLOBAL_DIR = "/home/node/.npm-global";
67
65
  // 命令执行时注入的 PATH:把 npm 全局 bin 放最前。
68
66
  const SANDBOX_PATH = `${NPM_GLOBAL_DIR}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`;
69
67
 
70
- /** Readable stream → Buffer。 */
71
- async function readableToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> {
72
- return new Promise((resolve, reject) => {
73
- const chunks: Buffer[] = [];
74
- stream.on("data", (chunk: Buffer | string) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
75
- stream.on("end", () => resolve(Buffer.concat(chunks)));
76
- stream.on("error", reject);
77
- });
78
- }
79
-
80
- /** 从单文件 tar 包里提取第一个 entry 的内容。 */
81
- async function extractFileFromTar(tarBuf: Buffer): Promise<Buffer> {
82
- return new Promise((resolve, reject) => {
83
- const extract = tar.extract();
84
- let found = false;
85
- extract.on("entry", (header, stream, next) => {
86
- if (!found) {
87
- found = true;
88
- const chunks: Buffer[] = [];
89
- stream.on("data", (c: Buffer) => chunks.push(c));
90
- stream.on("end", () => { resolve(Buffer.concat(chunks)); next(); });
91
- stream.on("error", reject);
92
- } else {
93
- stream.resume();
94
- next();
95
- }
96
- });
97
- extract.on("finish", () => { if (!found) reject(new Error("tar: no entries found")); });
98
- extract.on("error", reject);
99
- extract.end(tarBuf);
100
- });
101
- }
102
-
103
68
  /** 创建 Docker 沙箱的选项。 */
104
69
  export interface DockerSandboxOptions {
105
70
  /** 单条命令超时(毫秒)。 */
@@ -115,6 +80,7 @@ export interface DockerSandboxOptions {
115
80
  * 实现 ../types.ts 的 Sandbox 接口。
116
81
  */
117
82
  export class DockerSandbox implements Sandbox {
83
+ readonly workdir = CONTAINER_WORKDIR;
118
84
  readonly otlpHost = "host.docker.internal";
119
85
  private docker: Docker;
120
86
  private container: Docker.Container | null = null;
@@ -269,7 +235,7 @@ export class DockerSandbox implements Sandbox {
269
235
 
270
236
  return this.execCommand(cmd, args, {
271
237
  env,
272
- cwd: opts.cwd,
238
+ cwd: resolveSandboxPath(this.workdir, opts.cwd),
273
239
  user: isRoot ? ROOT_USER : SANDBOX_USER,
274
240
  });
275
241
  }
@@ -282,7 +248,7 @@ export class DockerSandbox implements Sandbox {
282
248
  ): Promise<CommandResult> {
283
249
  return this.execCommand(cmd, args, {
284
250
  env: opts.env,
285
- cwd: opts.cwd,
251
+ cwd: resolveSandboxPath(this.workdir, opts.cwd),
286
252
  user: ROOT_USER,
287
253
  });
288
254
  }
@@ -311,7 +277,7 @@ export class DockerSandbox implements Sandbox {
311
277
  Cmd: fullCmd,
312
278
  AttachStdout: true,
313
279
  AttachStderr: true,
314
- WorkingDir: opts.cwd ?? CONTAINER_WORKDIR,
280
+ WorkingDir: resolveSandboxPath(this.workdir, opts.cwd),
315
281
  Env: env,
316
282
  User: opts.user,
317
283
  });
@@ -319,29 +285,11 @@ export class DockerSandbox implements Sandbox {
319
285
  const stream = await exec.start({ hijack: true, stdin: false });
320
286
 
321
287
  return new Promise<CommandResult>((resolve, reject) => {
322
- // Docker 把 stdout/stderr 复用在同一条流里(8 字节头 + 载荷),需手动 demux
323
- // 头:[stream_type(1B), 0, 0, 0, size(4B 大端)];stream_type:1=stdout,2=stderr
324
- //
325
- // 关键:一帧可能被 Node 的可读流切到【多个 data 事件】里(尤其大输出,如 cat 一个
326
- // ~100KB 的文件),帧头 / 载荷都可能跨 chunk。所以必须跨 data 累积一个 leftover,
327
- // 只消费「已到齐的完整帧」,残帧留到下个 data —— 否则会在 chunk 边界丢字节 / 串帧,
328
- // 表现为 transcript 里随机损坏的行(曾导致 bub tape 的 tool_result/tool_call 被吞)。
329
- const stdoutChunks: Buffer[] = [];
330
- const stderrChunks: Buffer[] = [];
331
- let buffer: Buffer = Buffer.alloc(0); // 跨 data 累积的残帧;注解为 Buffer 以容纳 concat 的 ArrayBufferLike
332
-
333
- stream.on("data", (chunk: Buffer) => {
334
- buffer = buffer.length ? Buffer.concat([buffer, chunk]) : chunk;
335
- while (buffer.length >= 8) {
336
- const streamType = buffer[0];
337
- const size = buffer.readUInt32BE(4);
338
- if (buffer.length < 8 + size) break; // 载荷未到齐 → 等下个 data
339
- const payload = buffer.subarray(8, 8 + size);
340
- if (streamType === 2) stderrChunks.push(payload);
341
- else stdoutChunks.push(payload); // 1 / 0 / 未知 → 归 stdout
342
- buffer = buffer.subarray(8 + size);
343
- }
344
- });
288
+ // Docker 把 stdout/stderr 复用在同一条流里(8 字节头 + 载荷),需手动 demux;
289
+ // chunk 的帧累积逻辑见 docker-stream.ts createExecDemuxer
290
+ const demuxer = createExecDemuxer();
291
+
292
+ stream.on("data", (chunk: Buffer) => demuxer.push(chunk));
345
293
 
346
294
  // 超时:杀流并 reject。
347
295
  const timeoutId = setTimeout(() => {
@@ -351,8 +299,8 @@ export class DockerSandbox implements Sandbox {
351
299
 
352
300
  stream.on("end", async () => {
353
301
  clearTimeout(timeoutId);
354
- const stdout = Buffer.concat(stdoutChunks).toString("utf-8");
355
- const stderr = Buffer.concat(stderrChunks).toString("utf-8");
302
+ const stdout = demuxer.stdout();
303
+ const stderr = demuxer.stderr();
356
304
 
357
305
  try {
358
306
  const inspection = await exec.inspect();
@@ -391,16 +339,17 @@ export class DockerSandbox implements Sandbox {
391
339
 
392
340
  /** 读容器里的文件。 */
393
341
  async readFile(path: string): Promise<string> {
394
- const result = await this.runCommand("cat", [path]);
342
+ const absPath = resolveSandboxPath(this.workdir, path);
343
+ const result = await this.runCommand("cat", [absPath]);
395
344
  if (result.exitCode !== 0) {
396
- throw new Error(t("docker.readFileFailed", { path, stderr: result.stderr }));
345
+ throw new Error(t("docker.readFileFailed", { path: absPath, stderr: result.stderr }));
397
346
  }
398
347
  return result.stdout;
399
348
  }
400
349
 
401
350
  /** 判断容器里某文件是否存在。 */
402
351
  async fileExists(path: string): Promise<boolean> {
403
- const result = await this.runCommand("test", ["-f", path]);
352
+ const result = await this.runCommand("test", ["-f", resolveSandboxPath(this.workdir, path)]);
404
353
  return result.exitCode === 0;
405
354
  }
406
355
 
@@ -413,10 +362,8 @@ export class DockerSandbox implements Sandbox {
413
362
  const ignoreDirs = opts.ignoreDirs ?? DEFAULT_IGNORE_DIRS;
414
363
  const ignoreFiles = new Set(opts.ignoreFiles ?? DEFAULT_IGNORE_FILES);
415
364
 
416
- const dirPrune = ignoreDirs.map((d) => `-name '${d}'`).join(" -o ");
417
- const nameTests = extensions.map((e) => `-name '*.${e}'`).join(" -o ");
418
365
  const script =
419
- `find . \\( -type d \\( ${dirPrune} \\) \\) -prune -o -type f \\( ${nameTests} \\) -print | ` +
366
+ `${buildFindScript({ extensions, ignoreDirs })} | ` +
420
367
  `while IFS= read -r f; do printf '%s%s\\n' '${SOURCE_FILE_MARKER}' "$f"; cat "$f"; printf '\\n'; done`;
421
368
  const result = await this.runShell(script);
422
369
 
@@ -445,7 +392,7 @@ export class DockerSandbox implements Sandbox {
445
392
  }
446
393
 
447
394
  /** 用 tar 归档把文件灌进容器。 */
448
- async uploadFiles(files: SandboxFile[], targetDir = CONTAINER_WORKDIR): Promise<void> {
395
+ async uploadFiles(files: SandboxFile[], targetDir?: string): Promise<void> {
449
396
  if (!this.container) {
450
397
  throw new Error(t("docker.containerNotInitialized"));
451
398
  }
@@ -455,27 +402,25 @@ export class DockerSandbox implements Sandbox {
455
402
  }
456
403
 
457
404
  // 打 tar 包。
458
- const pack = tar.pack();
459
-
460
- for (const file of files) {
461
- const content =
462
- typeof file.content === "string" ? Buffer.from(file.content, "utf-8") : file.content;
405
+ const pack = packFilesToTar(
406
+ files.map((file) => ({
407
+ name: file.path,
408
+ content: typeof file.content === "string" ? Buffer.from(file.content, "utf-8") : file.content,
409
+ })),
410
+ );
463
411
 
464
- pack.entry({ name: file.path }, content);
465
- }
412
+ const targetPath = resolveSandboxPath(this.workdir, targetDir);
466
413
 
467
- pack.finalize();
468
-
469
- await this.runCommandAsRoot("mkdir", ["-p", targetDir]);
414
+ await this.runCommandAsRoot("mkdir", ["-p", targetPath]);
470
415
 
471
416
  // putArchive 以 root 身份解包到目标目录。
472
- await this.container.putArchive(pack, { path: targetDir });
417
+ await this.container.putArchive(pack, { path: targetPath });
473
418
 
474
419
  // 修正属主:putArchive 上传成 root,改回 node 用户,agent 才能编辑。
475
- await this.chownToSandboxUser(targetDir);
420
+ await this.chownToSandboxUser(targetPath);
476
421
  }
477
422
 
478
- async uploadDirectory(localDir: string, targetDir: string, opts: { ignore?: string[] } = {}): Promise<void> {
423
+ async uploadDirectory(localDir: string, targetDir?: string, opts: { ignore?: string[] } = {}): Promise<void> {
479
424
  const files = await collectLocalFiles(localDir, opts.ignore);
480
425
  await this.uploadFiles(files, targetDir);
481
426
  }
@@ -486,7 +431,7 @@ export class DockerSandbox implements Sandbox {
486
431
  */
487
432
  async downloadFile(path: string): Promise<Buffer> {
488
433
  if (!this.container) throw new Error(t("docker.containerNotInitialized"));
489
- const stream = await (this.container as Docker.Container).getArchive({ path });
434
+ const stream = await (this.container as Docker.Container).getArchive({ path: resolveSandboxPath(this.workdir, path) });
490
435
  const tarBuf = await readableToBuffer(stream as NodeJS.ReadableStream);
491
436
  return extractFileFromTar(tarBuf);
492
437
  }
@@ -497,10 +442,9 @@ export class DockerSandbox implements Sandbox {
497
442
  */
498
443
  async uploadFile(destPath: string, content: Buffer): Promise<void> {
499
444
  if (!this.container) throw new Error(t("docker.containerNotInitialized"));
500
- const pack = tar.pack();
501
- pack.entry({ name: basename(destPath) }, content);
502
- pack.finalize();
503
- await (this.container as Docker.Container).putArchive(pack, { path: dirname(destPath) });
445
+ const absPath = resolveSandboxPath(this.workdir, destPath);
446
+ const pack = packFilesToTar([{ name: basename(absPath), content }]);
447
+ await (this.container as Docker.Container).putArchive(pack, { path: dirname(absPath) });
504
448
  }
505
449
 
506
450
  /** 停止并清理容器(AutoRemove 负责销毁)。 */
@@ -515,25 +459,3 @@ export class DockerSandbox implements Sandbox {
515
459
  }
516
460
  }
517
461
  }
518
-
519
- async function collectLocalFiles(localDir: string, ignore: readonly string[] = []): Promise<SandboxFile[]> {
520
- const ignored = new Set(ignore);
521
- const out: SandboxFile[] = [];
522
- async function walk(dir: string): Promise<void> {
523
- for (const entry of await readdir(dir)) {
524
- if (ignored.has(entry)) continue;
525
- const abs = join(dir, entry);
526
- const st = await stat(abs);
527
- if (st.isDirectory()) {
528
- await walk(abs);
529
- } else if (st.isFile()) {
530
- out.push({
531
- path: relative(localDir, abs).split(sep).join("/"),
532
- content: await readFile(abs),
533
- });
534
- }
535
- }
536
- }
537
- await walk(localDir);
538
- return out;
539
- }