niceeval 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +164 -0
  2. package/README.zh.md +160 -0
  3. package/bin/niceeval.js +11 -0
  4. package/package.json +96 -0
  5. package/src/agents/bub.ts +223 -0
  6. package/src/agents/builtin.ts +6 -0
  7. package/src/agents/claude-code.ts +96 -0
  8. package/src/agents/codex.ts +122 -0
  9. package/src/agents/index.ts +25 -0
  10. package/src/agents/shared.ts +145 -0
  11. package/src/cli.ts +421 -0
  12. package/src/context/context.test.ts +96 -0
  13. package/src/context/context.ts +427 -0
  14. package/src/context/control-flow.ts +27 -0
  15. package/src/context/session.ts +124 -0
  16. package/src/define.ts +112 -0
  17. package/src/expect/index.ts +243 -0
  18. package/src/i18n/en.ts +131 -0
  19. package/src/i18n/index.ts +39 -0
  20. package/src/i18n/zh-CN.ts +132 -0
  21. package/src/index.ts +39 -0
  22. package/src/loaders/index.ts +56 -0
  23. package/src/o11y/cost.ts +57 -0
  24. package/src/o11y/derive.ts +304 -0
  25. package/src/o11y/otlp/canonical.ts +112 -0
  26. package/src/o11y/otlp/mappers/bub.ts +30 -0
  27. package/src/o11y/otlp/mappers/codex.ts +38 -0
  28. package/src/o11y/otlp/mappers/index.ts +25 -0
  29. package/src/o11y/otlp/parse.ts +330 -0
  30. package/src/o11y/otlp/receiver.ts +100 -0
  31. package/src/o11y/otlp/sandbox-receiver.ts +113 -0
  32. package/src/o11y/otlp/select.ts +82 -0
  33. package/src/o11y/parsers/bub.ts +240 -0
  34. package/src/o11y/parsers/claude-code.ts +270 -0
  35. package/src/o11y/parsers/codex.ts +480 -0
  36. package/src/o11y/parsers/index.ts +37 -0
  37. package/src/o11y/prices.json +9873 -0
  38. package/src/runner/discover.ts +77 -0
  39. package/src/runner/reporters/artifacts.ts +81 -0
  40. package/src/runner/reporters/console.ts +76 -0
  41. package/src/runner/reporters/index.ts +5 -0
  42. package/src/runner/reporters/json.ts +52 -0
  43. package/src/runner/reporters/live.ts +178 -0
  44. package/src/runner/reporters/table.ts +328 -0
  45. package/src/runner/run.ts +860 -0
  46. package/src/runner/sandbox-prep.ts +91 -0
  47. package/src/sandbox/checkpoint.ts +39 -0
  48. package/src/sandbox/docker.ts +539 -0
  49. package/src/sandbox/e2b.ts +203 -0
  50. package/src/sandbox/index.ts +25 -0
  51. package/src/sandbox/registry.ts +60 -0
  52. package/src/sandbox/resolve.ts +131 -0
  53. package/src/sandbox/source-files.ts +30 -0
  54. package/src/sandbox/vercel.ts +236 -0
  55. package/src/scoring/collector.ts +113 -0
  56. package/src/scoring/judge.ts +236 -0
  57. package/src/scoring/scoped.ts +289 -0
  58. package/src/scoring/verdict.ts +20 -0
  59. package/src/source-loc.ts +53 -0
  60. package/src/types.ts +913 -0
  61. package/src/util.test.ts +31 -0
  62. package/src/util.ts +50 -0
  63. package/src/view/app/App.tsx +189 -0
  64. package/src/view/app/components/AttemptModal.tsx +66 -0
  65. package/src/view/app/components/CodeView.tsx +272 -0
  66. package/src/view/app/components/CopyControls.tsx +89 -0
  67. package/src/view/app/components/ExperimentTable.tsx +266 -0
  68. package/src/view/app/components/GroupSelector.tsx +61 -0
  69. package/src/view/app/components/LazyArtifact.tsx +50 -0
  70. package/src/view/app/components/Trace.tsx +100 -0
  71. package/src/view/app/components/Transcript.tsx +130 -0
  72. package/src/view/app/components/primitives.tsx +43 -0
  73. package/src/view/app/components/ui/badge.tsx +21 -0
  74. package/src/view/app/components/ui/dialog.tsx +34 -0
  75. package/src/view/app/components/ui/tabs.tsx +30 -0
  76. package/src/view/app/i18n.ts +341 -0
  77. package/src/view/app/index.html +13 -0
  78. package/src/view/app/lib/cn.ts +7 -0
  79. package/src/view/app/lib/format.ts +73 -0
  80. package/src/view/app/lib/guards.ts +61 -0
  81. package/src/view/app/lib/outcome.ts +96 -0
  82. package/src/view/app/lib/rows.ts +63 -0
  83. package/src/view/app/lib/transcript-data.tsx +121 -0
  84. package/src/view/app/main.tsx +17 -0
  85. package/src/view/app/pages/RunsPage.tsx +83 -0
  86. package/src/view/app/pages/TracesPage.tsx +40 -0
  87. package/src/view/app/shared.ts +10 -0
  88. package/src/view/app/types.ts +114 -0
  89. package/src/view/app/vite.config.ts +26 -0
  90. package/src/view/client-dist/app.css +2 -0
  91. package/src/view/client-dist/app.js +56 -0
  92. package/src/view/index.ts +406 -0
  93. package/src/view/styles.css +1074 -0
  94. package/src/view/template.html +15 -0
@@ -0,0 +1,330 @@
1
+ // OTLP traces → niceeval TraceSpan[]。两种线编码都吃,无外部依赖:
2
+ // · OTLP/JSON ——— codex 的 trace_exporter protocol="json";id 是 hex、纳秒是十进制字符串。
3
+ // · OTLP/protobuf — bub-tapestore-otel(Python 的 HTTP 出口只有 protobuf)。
4
+ // 只抠瀑布图要的字段:traceId / spanId / parentSpanId / name / 起止纳秒 / status / attributes。
5
+ // span 的语义(span 名、属性约定)各 agent 不同,这里不解释,原样带给 view。
6
+
7
+ import type { JsonValue, TraceSpan } from "../../types.ts";
8
+
9
+ /** 按 Content-Type 选编码,失败一律回空数组(导出端不该因我们解析挂而报错)。 */
10
+ export function parseOtlpTraces(body: Buffer, contentType: string): TraceSpan[] {
11
+ try {
12
+ if (contentType.includes("json")) return parseJsonTraces(body.toString("utf8"));
13
+ return parseProtoTraces(body);
14
+ } catch {
15
+ return [];
16
+ }
17
+ }
18
+
19
+ // ───────────────────────── 纳秒字符串/数字 → epoch 毫秒 ─────────────────────────
20
+
21
+ function nanoToMs(nano: string | number | bigint | undefined): number {
22
+ if (nano === undefined || nano === null) return 0;
23
+ try {
24
+ return Number(BigInt(nano) / 1_000_000n);
25
+ } catch {
26
+ return 0;
27
+ }
28
+ }
29
+
30
+ function statusFromCode(code: number | string | undefined): TraceSpan["status"] {
31
+ // OTLP StatusCode:0 UNSET / 1 OK / 2 ERROR(JSON 里可能是数字或枚举名)。
32
+ if (code === 2 || code === "STATUS_CODE_ERROR") return "error";
33
+ if (code === 1 || code === "STATUS_CODE_OK") return "ok";
34
+ return "unset";
35
+ }
36
+
37
+ // ───────────────────────── OTLP/JSON ─────────────────────────
38
+
39
+ interface JsonAnyValue {
40
+ stringValue?: string;
41
+ boolValue?: boolean;
42
+ intValue?: string | number;
43
+ doubleValue?: number;
44
+ arrayValue?: { values?: JsonAnyValue[] };
45
+ kvlistValue?: { values?: { key?: string; value?: JsonAnyValue }[] };
46
+ bytesValue?: string;
47
+ }
48
+
49
+ function jsonAnyValue(v: JsonAnyValue | undefined): JsonValue {
50
+ if (!v) return null;
51
+ if (v.stringValue !== undefined) return v.stringValue;
52
+ if (v.boolValue !== undefined) return v.boolValue;
53
+ if (v.intValue !== undefined) return Number(v.intValue);
54
+ if (v.doubleValue !== undefined) return v.doubleValue;
55
+ if (v.bytesValue !== undefined) return v.bytesValue;
56
+ if (v.arrayValue) return (v.arrayValue.values ?? []).map(jsonAnyValue);
57
+ if (v.kvlistValue) {
58
+ const obj: Record<string, JsonValue> = {};
59
+ for (const kv of v.kvlistValue.values ?? []) if (kv.key) obj[kv.key] = jsonAnyValue(kv.value);
60
+ return obj;
61
+ }
62
+ return null;
63
+ }
64
+
65
+ function parseJsonTraces(text: string): TraceSpan[] {
66
+ if (!text.trim()) return [];
67
+ const root = JSON.parse(text) as {
68
+ resourceSpans?: { scopeSpans?: { spans?: JsonSpan[] }[] }[];
69
+ };
70
+ const out: TraceSpan[] = [];
71
+ for (const rs of root.resourceSpans ?? []) {
72
+ for (const ss of rs.scopeSpans ?? []) {
73
+ for (const s of ss.spans ?? []) out.push(jsonSpan(s));
74
+ }
75
+ }
76
+ return out;
77
+ }
78
+
79
+ interface JsonSpan {
80
+ traceId?: string;
81
+ spanId?: string;
82
+ parentSpanId?: string;
83
+ name?: string;
84
+ startTimeUnixNano?: string;
85
+ endTimeUnixNano?: string;
86
+ status?: { code?: number | string; message?: string };
87
+ attributes?: { key?: string; value?: JsonAnyValue }[];
88
+ }
89
+
90
+ function jsonSpan(s: JsonSpan): TraceSpan {
91
+ const attributes: Record<string, JsonValue> = {};
92
+ for (const a of s.attributes ?? []) if (a.key) attributes[a.key] = jsonAnyValue(a.value);
93
+ return {
94
+ traceId: s.traceId ?? "",
95
+ spanId: s.spanId ?? "",
96
+ parentSpanId: s.parentSpanId || undefined,
97
+ name: s.name ?? "span",
98
+ startMs: nanoToMs(s.startTimeUnixNano),
99
+ endMs: nanoToMs(s.endTimeUnixNano),
100
+ status: statusFromCode(s.status?.code),
101
+ attributes: Object.keys(attributes).length ? attributes : undefined,
102
+ };
103
+ }
104
+
105
+ // ───────────────────────── OTLP/protobuf ─────────────────────────
106
+ // 手写一个够用的 protobuf 读取器 + OTLP trace 消息游走(字段号见 opentelemetry-proto)。
107
+
108
+ class Reader {
109
+ pos = 0;
110
+ constructor(
111
+ private readonly buf: Buffer,
112
+ private readonly end = buf.length,
113
+ ) {}
114
+ get eof(): boolean {
115
+ return this.pos >= this.end;
116
+ }
117
+ varint(): bigint {
118
+ let shift = 0n;
119
+ let result = 0n;
120
+ for (;;) {
121
+ const b = this.buf[this.pos++];
122
+ result |= BigInt(b & 0x7f) << shift;
123
+ if ((b & 0x80) === 0) break;
124
+ shift += 7n;
125
+ }
126
+ return result;
127
+ }
128
+ fixed64(): bigint {
129
+ const lo = BigInt(this.buf.readUInt32LE(this.pos));
130
+ const hi = BigInt(this.buf.readUInt32LE(this.pos + 4));
131
+ this.pos += 8;
132
+ return (hi << 32n) | lo;
133
+ }
134
+ double(): number {
135
+ const v = this.buf.readDoubleLE(this.pos);
136
+ this.pos += 8;
137
+ return v;
138
+ }
139
+ bytes(): Buffer {
140
+ const len = Number(this.varint());
141
+ const b = this.buf.subarray(this.pos, this.pos + len);
142
+ this.pos += len;
143
+ return b;
144
+ }
145
+ /** 跳过当前 wire type 的一个值(遇到不认识的字段)。 */
146
+ skip(wire: number): void {
147
+ if (wire === 0) this.varint();
148
+ else if (wire === 1) this.pos += 8;
149
+ else if (wire === 2) {
150
+ // 注意:必须先读完 length varint(它会推进 pos),再加 len。
151
+ // 写成 `this.pos += Number(this.varint())` 有 bug:+= 会先取旧 pos,丢掉 length 前缀字节 → 错位。
152
+ const len = Number(this.varint());
153
+ this.pos += len;
154
+ } else if (wire === 5) this.pos += 4;
155
+ }
156
+ }
157
+
158
+ /** 遍历一个 message 的字段;cb 负责消费当前字段值(读或 r.skip(wire))。 */
159
+ function eachField(buf: Buffer, cb: (field: number, wire: number, r: Reader) => void): void {
160
+ const r = new Reader(buf);
161
+ while (!r.eof) {
162
+ const tag = Number(r.varint());
163
+ cb(tag >> 3, tag & 7, r);
164
+ }
165
+ }
166
+
167
+ function parseProtoTraces(buf: Buffer): TraceSpan[] {
168
+ const spans: TraceSpan[] = [];
169
+ // ExportTraceServiceRequest { 1: repeated ResourceSpans }
170
+ eachField(buf, (field, wire, r) => {
171
+ if (field === 1 && wire === 2) protoResourceSpans(r.bytes(), spans);
172
+ else r.skip(wire);
173
+ });
174
+ return spans;
175
+ }
176
+
177
+ function protoResourceSpans(buf: Buffer, spans: TraceSpan[]): void {
178
+ // ResourceSpans { 2: repeated ScopeSpans }(忽略 resource 属性)
179
+ eachField(buf, (field, wire, r) => {
180
+ if (field === 2 && wire === 2) protoScopeSpans(r.bytes(), spans);
181
+ else r.skip(wire);
182
+ });
183
+ }
184
+
185
+ function protoScopeSpans(buf: Buffer, spans: TraceSpan[]): void {
186
+ // ScopeSpans { 2: repeated Span }
187
+ eachField(buf, (field, wire, r) => {
188
+ if (field === 2 && wire === 2) spans.push(protoSpan(r.bytes()));
189
+ else r.skip(wire);
190
+ });
191
+ }
192
+
193
+ function protoSpan(buf: Buffer): TraceSpan {
194
+ let traceId = "";
195
+ let spanId = "";
196
+ let parentSpanId = "";
197
+ let name = "span";
198
+ let startNano = 0n;
199
+ let endNano = 0n;
200
+ let status: TraceSpan["status"] = "unset";
201
+ const attributes: Record<string, JsonValue> = {};
202
+ // Span { 1:trace_id 2:span_id 4:parent_span_id 5:name 7:start(fixed64)
203
+ // 8:end(fixed64) 9:repeated KeyValue attrs 15:Status }
204
+ eachField(buf, (field, wire, r) => {
205
+ switch (field) {
206
+ case 1:
207
+ traceId = r.bytes().toString("hex");
208
+ break;
209
+ case 2:
210
+ spanId = r.bytes().toString("hex");
211
+ break;
212
+ case 4:
213
+ parentSpanId = r.bytes().toString("hex");
214
+ break;
215
+ case 5:
216
+ name = r.bytes().toString("utf8");
217
+ break;
218
+ case 7:
219
+ if (wire === 1) startNano = r.fixed64();
220
+ else r.skip(wire);
221
+ break;
222
+ case 8:
223
+ if (wire === 1) endNano = r.fixed64();
224
+ else r.skip(wire);
225
+ break;
226
+ case 9:
227
+ if (wire === 2) {
228
+ const kv = protoKeyValue(r.bytes());
229
+ if (kv.key) attributes[kv.key] = kv.value;
230
+ } else r.skip(wire);
231
+ break;
232
+ case 15:
233
+ if (wire === 2) status = protoStatus(r.bytes());
234
+ else r.skip(wire);
235
+ break;
236
+ default:
237
+ r.skip(wire);
238
+ }
239
+ });
240
+ return {
241
+ traceId,
242
+ spanId,
243
+ parentSpanId: parentSpanId || undefined,
244
+ name,
245
+ startMs: nanoToMs(startNano),
246
+ endMs: nanoToMs(endNano),
247
+ status,
248
+ attributes: Object.keys(attributes).length ? attributes : undefined,
249
+ };
250
+ }
251
+
252
+ function protoStatus(buf: Buffer): TraceSpan["status"] {
253
+ let code = 0;
254
+ // Status { 2:message 3:code(enum varint) }
255
+ eachField(buf, (field, wire, r) => {
256
+ if (field === 3 && wire === 0) code = Number(r.varint());
257
+ else r.skip(wire);
258
+ });
259
+ return statusFromCode(code);
260
+ }
261
+
262
+ function protoKeyValue(buf: Buffer): { key?: string; value: JsonValue } {
263
+ let key: string | undefined;
264
+ let value: JsonValue = null;
265
+ // KeyValue { 1:key 2:AnyValue }
266
+ eachField(buf, (field, wire, r) => {
267
+ if (field === 1 && wire === 2) key = r.bytes().toString("utf8");
268
+ else if (field === 2 && wire === 2) value = protoAnyValue(r.bytes());
269
+ else r.skip(wire);
270
+ });
271
+ return { key, value };
272
+ }
273
+
274
+ function protoAnyValue(buf: Buffer): JsonValue {
275
+ let value: JsonValue = null;
276
+ // AnyValue { 1:string 2:bool 3:int(varint) 4:double(fixed64) 5:array 6:kvlist 7:bytes }
277
+ eachField(buf, (field, wire, r) => {
278
+ switch (field) {
279
+ case 1:
280
+ value = r.bytes().toString("utf8");
281
+ break;
282
+ case 2:
283
+ value = Number(r.varint()) !== 0;
284
+ break;
285
+ case 3:
286
+ value = Number(r.varint());
287
+ break;
288
+ case 4:
289
+ if (wire === 1) value = r.double();
290
+ else r.skip(wire);
291
+ break;
292
+ case 5:
293
+ if (wire === 2) value = protoArray(r.bytes());
294
+ else r.skip(wire);
295
+ break;
296
+ case 6:
297
+ if (wire === 2) value = protoKvList(r.bytes());
298
+ else r.skip(wire);
299
+ break;
300
+ case 7:
301
+ value = r.bytes().toString("utf8");
302
+ break;
303
+ default:
304
+ r.skip(wire);
305
+ }
306
+ });
307
+ return value;
308
+ }
309
+
310
+ function protoArray(buf: Buffer): JsonValue {
311
+ const out: JsonValue[] = [];
312
+ // ArrayValue { 1:repeated AnyValue }
313
+ eachField(buf, (field, wire, r) => {
314
+ if (field === 1 && wire === 2) out.push(protoAnyValue(r.bytes()));
315
+ else r.skip(wire);
316
+ });
317
+ return out;
318
+ }
319
+
320
+ function protoKvList(buf: Buffer): JsonValue {
321
+ const obj: Record<string, JsonValue> = {};
322
+ // KeyValueList { 1:repeated KeyValue }
323
+ eachField(buf, (field, wire, r) => {
324
+ if (field === 1 && wire === 2) {
325
+ const kv = protoKeyValue(r.bytes());
326
+ if (kv.key) obj[kv.key] = kv.value;
327
+ } else r.skip(wire);
328
+ });
329
+ return obj;
330
+ }
@@ -0,0 +1,100 @@
1
+ // 本机 OTLP/HTTP traces 接收器:每个沙箱起一个,容器里的 agent 把 OTLP 导出到它。
2
+ // 监听 0.0.0.0 的临时端口(容器经 host.docker.internal 回连宿主),收到的 span 攒着,
3
+ // 跑完由运行器一次性 collect 挂到 EvalResult.trace。只认 POST .../v1/traces。
4
+
5
+ import { createServer } from "node:http";
6
+ import { gunzipSync } from "node:zlib";
7
+ import { Effect } from "effect";
8
+ import type { TraceSpan } from "../../types.ts";
9
+ import { parseOtlpTraces } from "./parse.ts";
10
+
11
+ export interface TraceReceiver {
12
+ /** agent 应导出到的完整端点(host 由后端定:docker → host.docker.internal)。 */
13
+ endpoint(host: string): string;
14
+ /** 目前为止收到并解析出的全部 span(副本)。 */
15
+ collect(): TraceSpan[];
16
+ /** 给在途的最后一批导出留点落地时间(无新 span 持续 quietMs 即返回,至多 maxMs)。 */
17
+ settle(quietMs: number, maxMs: number): Promise<void>;
18
+ close(): Promise<void>;
19
+ }
20
+
21
+ /**
22
+ * 创建 OTLP 接收器,并把 close() 注册为 Scope 回收动作(免端口泄漏)。
23
+ * 在 Effect.scoped / Effect.gen 里 yield* 即可。
24
+ */
25
+ export function createTraceReceiver() {
26
+ return Effect.acquireRelease(Effect.promise(makeReceiver), (r) =>
27
+ Effect.promise(() => r.close().catch(() => {})),
28
+ );
29
+ }
30
+
31
+ async function makeReceiver(): Promise<TraceReceiver> {
32
+ const spans: TraceSpan[] = [];
33
+ let lastAt = 0;
34
+
35
+ const server = createServer((req, res) => {
36
+ if (req.method !== "POST") {
37
+ res.writeHead(405).end();
38
+ return;
39
+ }
40
+ const chunks: Buffer[] = [];
41
+ req.on("data", (c: Buffer) => chunks.push(c));
42
+ req.on("end", () => {
43
+ const ct = req.headers["content-type"] ?? "";
44
+ try {
45
+ let body = Buffer.concat(chunks);
46
+ // OTLP 导出端可能 gzip(OTEL_EXPORTER_OTLP_COMPRESSION 或默认开)。按 header 或 gzip 魔数解压。
47
+ const ce = String(req.headers["content-encoding"] ?? "");
48
+ if (ce.includes("gzip") || (body[0] === 0x1f && body[1] === 0x8b)) {
49
+ try {
50
+ body = gunzipSync(body);
51
+ } catch {
52
+ // 不是合法 gzip 就按原样试
53
+ }
54
+ }
55
+ const parsed = parseOtlpTraces(body, ct);
56
+ if (parsed.length) {
57
+ spans.push(...parsed);
58
+ lastAt = Date.now();
59
+ }
60
+ } catch {
61
+ // 解析失败不回 5xx,免得导出端重试刷屏。
62
+ }
63
+ // OTLP 成功响应:空 ExportTraceServiceResponse(JSON {} / protobuf 空 body)。
64
+ if (ct.includes("json")) {
65
+ res.writeHead(200, { "content-type": "application/json" }).end("{}");
66
+ } else {
67
+ res.writeHead(200, { "content-type": "application/x-protobuf" }).end(Buffer.alloc(0));
68
+ }
69
+ });
70
+ req.on("error", () => res.writeHead(400).end());
71
+ });
72
+
73
+ const port = await new Promise<number>((resolve, reject) => {
74
+ server.once("error", reject);
75
+ // 0.0.0.0:容器经 host-gateway / host.docker.internal 回连宿主,不能只听 127.0.0.1。
76
+ server.listen(0, "0.0.0.0", () => {
77
+ const a = server.address();
78
+ resolve(typeof a === "object" && a ? a.port : 0);
79
+ });
80
+ });
81
+
82
+ return {
83
+ endpoint: (host) => `http://${host}:${port}/v1/traces`,
84
+ collect: () => spans.slice(),
85
+ async settle(quietMs, maxMs) {
86
+ // 等到「收到过 span 且静默 quietMs」或「整体超 maxMs」。还没收到任何 span 时
87
+ // 也一直等到 maxMs —— 最后一批导出可能正在途中(进程刚退、POST 还没到)。
88
+ const deadline = Date.now() + maxMs;
89
+ for (;;) {
90
+ if (Date.now() >= deadline) return;
91
+ if (lastAt !== 0 && Date.now() - lastAt >= quietMs) return;
92
+ await new Promise((r) => setTimeout(r, 50));
93
+ }
94
+ },
95
+ close: () =>
96
+ new Promise((resolve) => {
97
+ server.close(() => resolve());
98
+ }),
99
+ };
100
+ }
@@ -0,0 +1,113 @@
1
+ // 在沙箱内部起一个轻量 OTLP/HTTP 接收器,供 e2b / vercel 等远程沙箱使用。
2
+ // 流程:
3
+ // 1. writeFiles 把 collector 脚本上传到沙箱
4
+ // 2. runShell 在后台启动它(node ... & echo $!),拿 PID
5
+ // 3. agent 往 http://127.0.0.1:4318/v1/traces 发 span(sandbox 内 localhost)
6
+ // 4. settle() 等 agent 退出后下载 spans 文件并解析,缓存到内存
7
+ // 5. collect() 返回缓存(同步,与本地 receiver 接口一致)
8
+ // 6. close() 尝试 kill PID(沙箱本身也会停,所以 best-effort)
9
+
10
+ import { Effect } from "effect";
11
+ import type { TraceSpan } from "../../types.ts";
12
+ import type { Sandbox } from "../../types.ts";
13
+ import type { TraceReceiver } from "./receiver.ts";
14
+ import { parseOtlpTraces } from "./parse.ts";
15
+
16
+ const COLLECTOR_PORT = 4318;
17
+ const COLLECTOR_PATH = "/tmp/.niceeval-otlp-collector.cjs";
18
+ const SPANS_PATH = "/tmp/.niceeval-otlp-spans.jsonl";
19
+
20
+ // collector 脚本:纯 Node.js CJS,无外部依赖。
21
+ // 每收一个 OTLP/HTTP 请求就把 { ct, body(base64) } 追加写一行到 SPANS_PATH,
22
+ // 同时处理 gzip 解压——这样 host 侧直接复用 parseOtlpTraces 解析。
23
+ const COLLECTOR_SCRIPT = /* js */ `
24
+ 'use strict';
25
+ const http = require('http');
26
+ const fs = require('fs');
27
+ const zlib = require('zlib');
28
+ const OUT = ${JSON.stringify(SPANS_PATH)};
29
+ const server = http.createServer((req, res) => {
30
+ if (req.method !== 'POST') { res.writeHead(405).end(); return; }
31
+ const chunks = [];
32
+ req.on('data', c => chunks.push(c));
33
+ req.on('end', () => {
34
+ let body = Buffer.concat(chunks);
35
+ const ct = req.headers['content-type'] || '';
36
+ const ce = req.headers['content-encoding'] || '';
37
+ if (ce.includes('gzip') || (body[0] === 0x1f && body[1] === 0x8b)) {
38
+ try { body = zlib.gunzipSync(body); } catch {}
39
+ }
40
+ try { fs.appendFileSync(OUT, JSON.stringify({ ct, b: body.toString('base64') }) + '\\n'); } catch {}
41
+ if (ct.includes('json')) {
42
+ res.writeHead(200, { 'content-type': 'application/json' }).end('{}');
43
+ } else {
44
+ res.writeHead(200, { 'content-type': 'application/x-protobuf' }).end(Buffer.alloc(0));
45
+ }
46
+ });
47
+ req.on('error', () => res.writeHead(400).end());
48
+ });
49
+ server.listen(${COLLECTOR_PORT}, '127.0.0.1');
50
+ `;
51
+
52
+ export function createInSandboxTraceReceiver(sandbox: Sandbox) {
53
+ return Effect.acquireRelease(
54
+ Effect.promise(() => makeInSandboxReceiver(sandbox)),
55
+ (r) => Effect.promise(() => r.close().catch(() => {})),
56
+ );
57
+ }
58
+
59
+ async function makeInSandboxReceiver(sandbox: Sandbox): Promise<TraceReceiver> {
60
+ let cached: TraceSpan[] = [];
61
+
62
+ // 上传 collector 脚本
63
+ await sandbox.writeFiles({ [COLLECTOR_PATH]: COLLECTOR_SCRIPT });
64
+
65
+ // 后台启动,拿 PID(shell 的 & 返回后台 PID;echo $! 给我们)
66
+ const startResult = await sandbox.runShell(
67
+ `node ${COLLECTOR_PATH} >/tmp/.niceeval-otlp-collector.log 2>&1 & echo $!`,
68
+ );
69
+ const pid = parseInt(startResult.stdout.trim(), 10);
70
+
71
+ return {
72
+ endpoint: (_host) => `http://127.0.0.1:${COLLECTOR_PORT}/v1/traces`,
73
+
74
+ collect: () => cached.slice(),
75
+
76
+ // agent 结束后调:等片刻(让 OTEL exporter 把最后几个 batch flush 完),再下载解析。
77
+ async settle(quietMs, maxMs) {
78
+ const wait = Math.min(quietMs + 300, maxMs);
79
+ await new Promise((r) => setTimeout(r, wait));
80
+ try {
81
+ const raw = await sandbox.downloadFile(SPANS_PATH);
82
+ cached = parseSpansFile(raw);
83
+ } catch {
84
+ // 没有 spans 文件(agent 没发任何 trace)→ 保留空数组
85
+ }
86
+ },
87
+
88
+ async close() {
89
+ if (Number.isFinite(pid) && pid > 0) {
90
+ // best-effort:沙箱停止时进程也会消失,这里只是提前清理
91
+ await sandbox.runShell(`kill ${pid} 2>/dev/null || true`).catch(() => {});
92
+ }
93
+ },
94
+ };
95
+ }
96
+
97
+ // SPANS_PATH 每行一个 { ct: string; b: string(base64) }
98
+ function parseSpansFile(raw: Buffer): TraceSpan[] {
99
+ const spans: TraceSpan[] = [];
100
+ const text = raw.toString("utf8");
101
+ for (const line of text.split("\n")) {
102
+ const trimmed = line.trim();
103
+ if (!trimmed) continue;
104
+ try {
105
+ const { ct, b } = JSON.parse(trimmed) as { ct: string; b: string };
106
+ const body = Buffer.from(b, "base64");
107
+ spans.push(...parseOtlpTraces(body, ct));
108
+ } catch {
109
+ // 跳过损坏行
110
+ }
111
+ }
112
+ return spans;
113
+ }
@@ -0,0 +1,82 @@
1
+ // 从一坨 OTLP span 里挑「语义」span:agent 真正做的事(回合 / 模型调用 / 工具调用),
2
+ // 丢掉每-chunk / 每-item 的内部 instrument span 与建表/配置类 plumbing。
3
+ //
4
+ // 为什么要挑:像 codex 这种 Rust agent 的 OTLP 把内部 tracing 全导出来了 —— 一个 eval 上万条
5
+ // span(handle_responses/receiving/append_items…),直接落盘会把 summary.json 撑到几十 MB、
6
+ // view 渲染几万行。挑完只剩百来条有意义的,瀑布图才读得动。
7
+ //
8
+ // 判定已经是【纯通用】的了:span 早在本步之前经 mapper 归一到 canonical(每-agent mapper 给每条
9
+ // span 定了 SpanKind,见 otlp/mappers/),所以这里不再认任何 agent 的原生 span 名 —— 只看 kind:
10
+ // 留下 kind ≠ "other" 的(回合 / 模型 / 工具 / 子 agent),丢掉 "other"(plumbing / 每-chunk 噪声)。
11
+ // 兜底:某 agent 没 mapper、kind 全 "other" 时,回落到「按 span 名频率丢 firehose」。
12
+
13
+ import type { ToolCall, TraceSpan, JsonValue } from "../../types.ts";
14
+
15
+ /** 不大的 trace 整段保留(没有 firehose 要对付)。 */
16
+ const SMALL_TRACE = 150;
17
+ /** 单个 span 名在一次运行里出现这么多次,视为每-chunk/每-item 内部噪声(仅无-mapper 兜底路径用)。 */
18
+ const FIREHOSE_FREQ = 80;
19
+ /** 过滤后仍超这个数,再按耗时降序硬截断兜底。 */
20
+ const HARD_CAP = 1000;
21
+
22
+ /**
23
+ * 选出要保留并落盘的 span。
24
+ * 小 trace 原样返回;大 trace 按 canonical kind 过滤(kind ≠ "other" 留);
25
+ * 没 mapper(kind 全 "other")则回落到 firehose 频率过滤;仍过多再按耗时硬截断。
26
+ * 最后一律按起点排序,view 直接当瀑布图渲染。
27
+ */
28
+ export function selectTraceSpans(spans: TraceSpan[]): TraceSpan[] {
29
+ if (spans.length <= SMALL_TRACE) return spans.slice().sort((a, b) => a.startMs - b.startMs);
30
+
31
+ // 主路径:按 canonical kind 留语义 span(agent 特定知识在 mapper 里,这里纯通用)。
32
+ let kept = spans.filter((sp) => sp.kind && sp.kind !== "other");
33
+
34
+ // 兜底:没 mapper / 全没识别(kind 全空或 "other")—— 退回按 span 名频率丢 firehose。
35
+ if (kept.length === 0) {
36
+ const freq: Record<string, number> = {};
37
+ for (const sp of spans) freq[sp.name] = (freq[sp.name] ?? 0) + 1;
38
+ kept = spans.filter((sp) => freq[sp.name] <= FIREHOSE_FREQ);
39
+ if (kept.length === 0) kept = spans;
40
+ }
41
+
42
+ if (kept.length > HARD_CAP) {
43
+ kept = kept
44
+ .slice()
45
+ .sort((a, b) => b.endMs - b.startMs - (a.endMs - a.startMs))
46
+ .slice(0, HARD_CAP);
47
+ }
48
+ return kept.sort((a, b) => a.startMs - b.startMs);
49
+ }
50
+
51
+ /** I/O 文本上限:文件内容/命令输出可能很大,截一下别把 trace 撑爆。 */
52
+ const IO_MAX = 4000;
53
+
54
+ function ioText(v: unknown): string {
55
+ const s = typeof v === "string" ? v : JSON.stringify(v);
56
+ if (s === undefined) return "";
57
+ return s.length > IO_MAX ? s.slice(0, IO_MAX) + `…(+${s.length - IO_MAX})` : s;
58
+ }
59
+
60
+ /**
61
+ * 给工具执行 span 补上真实「入参/出参」:codex 等的 OTLP span 只带 tool_name/call_id,
62
+ * 命令文本与输出在 stdout transcript(events)里 —— 按 call_id 把 deriveRunFacts 的
63
+ * ToolCall.input/output join 到对应 span 的 attributes(io.input / io.output / io.tool)。
64
+ * 没匹配上的 span 原样返回。
65
+ */
66
+ export function enrichTraceWithIO(spans: TraceSpan[], toolCalls: readonly ToolCall[]): TraceSpan[] {
67
+ const byCall = new Map<string, ToolCall>();
68
+ for (const tc of toolCalls) if (tc.callId) byCall.set(tc.callId, tc);
69
+ if (byCall.size === 0) return spans;
70
+
71
+ return spans.map((sp) => {
72
+ const cid = sp.attributes?.call_id;
73
+ const tc = typeof cid === "string" ? byCall.get(cid) : undefined;
74
+ if (!tc) return sp;
75
+ const attributes: Record<string, JsonValue> = { ...sp.attributes };
76
+ if (tc.originalName) attributes["io.tool"] = tc.originalName;
77
+ if (tc.input !== undefined && tc.input !== null) attributes["io.input"] = ioText(tc.input);
78
+ if (tc.output !== undefined && tc.output !== null) attributes["io.output"] = ioText(tc.output);
79
+ if (tc.status) attributes["io.status"] = tc.status;
80
+ return { ...sp, attributes };
81
+ });
82
+ }