niceeval 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +164 -0
  2. package/README.zh.md +160 -0
  3. package/bin/niceeval.js +11 -0
  4. package/package.json +96 -0
  5. package/src/agents/bub.ts +223 -0
  6. package/src/agents/builtin.ts +6 -0
  7. package/src/agents/claude-code.ts +96 -0
  8. package/src/agents/codex.ts +122 -0
  9. package/src/agents/index.ts +25 -0
  10. package/src/agents/shared.ts +145 -0
  11. package/src/cli.ts +421 -0
  12. package/src/context/context.test.ts +96 -0
  13. package/src/context/context.ts +427 -0
  14. package/src/context/control-flow.ts +27 -0
  15. package/src/context/session.ts +124 -0
  16. package/src/define.ts +112 -0
  17. package/src/expect/index.ts +243 -0
  18. package/src/i18n/en.ts +131 -0
  19. package/src/i18n/index.ts +39 -0
  20. package/src/i18n/zh-CN.ts +132 -0
  21. package/src/index.ts +39 -0
  22. package/src/loaders/index.ts +56 -0
  23. package/src/o11y/cost.ts +57 -0
  24. package/src/o11y/derive.ts +304 -0
  25. package/src/o11y/otlp/canonical.ts +112 -0
  26. package/src/o11y/otlp/mappers/bub.ts +30 -0
  27. package/src/o11y/otlp/mappers/codex.ts +38 -0
  28. package/src/o11y/otlp/mappers/index.ts +25 -0
  29. package/src/o11y/otlp/parse.ts +330 -0
  30. package/src/o11y/otlp/receiver.ts +100 -0
  31. package/src/o11y/otlp/sandbox-receiver.ts +113 -0
  32. package/src/o11y/otlp/select.ts +82 -0
  33. package/src/o11y/parsers/bub.ts +240 -0
  34. package/src/o11y/parsers/claude-code.ts +270 -0
  35. package/src/o11y/parsers/codex.ts +480 -0
  36. package/src/o11y/parsers/index.ts +37 -0
  37. package/src/o11y/prices.json +9873 -0
  38. package/src/runner/discover.ts +77 -0
  39. package/src/runner/reporters/artifacts.ts +81 -0
  40. package/src/runner/reporters/console.ts +76 -0
  41. package/src/runner/reporters/index.ts +5 -0
  42. package/src/runner/reporters/json.ts +52 -0
  43. package/src/runner/reporters/live.ts +178 -0
  44. package/src/runner/reporters/table.ts +328 -0
  45. package/src/runner/run.ts +860 -0
  46. package/src/runner/sandbox-prep.ts +91 -0
  47. package/src/sandbox/checkpoint.ts +39 -0
  48. package/src/sandbox/docker.ts +539 -0
  49. package/src/sandbox/e2b.ts +203 -0
  50. package/src/sandbox/index.ts +25 -0
  51. package/src/sandbox/registry.ts +60 -0
  52. package/src/sandbox/resolve.ts +131 -0
  53. package/src/sandbox/source-files.ts +30 -0
  54. package/src/sandbox/vercel.ts +236 -0
  55. package/src/scoring/collector.ts +113 -0
  56. package/src/scoring/judge.ts +236 -0
  57. package/src/scoring/scoped.ts +289 -0
  58. package/src/scoring/verdict.ts +20 -0
  59. package/src/source-loc.ts +53 -0
  60. package/src/types.ts +913 -0
  61. package/src/util.test.ts +31 -0
  62. package/src/util.ts +50 -0
  63. package/src/view/app/App.tsx +189 -0
  64. package/src/view/app/components/AttemptModal.tsx +66 -0
  65. package/src/view/app/components/CodeView.tsx +272 -0
  66. package/src/view/app/components/CopyControls.tsx +89 -0
  67. package/src/view/app/components/ExperimentTable.tsx +266 -0
  68. package/src/view/app/components/GroupSelector.tsx +61 -0
  69. package/src/view/app/components/LazyArtifact.tsx +50 -0
  70. package/src/view/app/components/Trace.tsx +100 -0
  71. package/src/view/app/components/Transcript.tsx +130 -0
  72. package/src/view/app/components/primitives.tsx +43 -0
  73. package/src/view/app/components/ui/badge.tsx +21 -0
  74. package/src/view/app/components/ui/dialog.tsx +34 -0
  75. package/src/view/app/components/ui/tabs.tsx +30 -0
  76. package/src/view/app/i18n.ts +341 -0
  77. package/src/view/app/index.html +13 -0
  78. package/src/view/app/lib/cn.ts +7 -0
  79. package/src/view/app/lib/format.ts +73 -0
  80. package/src/view/app/lib/guards.ts +61 -0
  81. package/src/view/app/lib/outcome.ts +96 -0
  82. package/src/view/app/lib/rows.ts +63 -0
  83. package/src/view/app/lib/transcript-data.tsx +121 -0
  84. package/src/view/app/main.tsx +17 -0
  85. package/src/view/app/pages/RunsPage.tsx +83 -0
  86. package/src/view/app/pages/TracesPage.tsx +40 -0
  87. package/src/view/app/shared.ts +10 -0
  88. package/src/view/app/types.ts +114 -0
  89. package/src/view/app/vite.config.ts +26 -0
  90. package/src/view/client-dist/app.css +2 -0
  91. package/src/view/client-dist/app.js +56 -0
  92. package/src/view/index.ts +406 -0
  93. package/src/view/styles.css +1074 -0
  94. package/src/view/template.html +15 -0
@@ -0,0 +1,57 @@
1
+ // 价格表兜底估算:agent 没带回网关实测成本时,用 token 用量 × vendored 单价估一个。
2
+ // 数据来自 src/o11y/prices.json(models.dev,见 scripts/sync-prices.ts);per-1M USD。
3
+ //
4
+ // 与 types.ts 的约定一致:usage.costUSD(实测)优先,这里只在缺实测时兜底,
5
+ // 查不到价就返回 undefined —— 显示 "—" 而不是骗人的 $0。
6
+
7
+ import { readFileSync } from "node:fs";
8
+ import { fileURLToPath } from "node:url";
9
+ import type { Usage } from "../types.ts";
10
+
11
+ interface Price {
12
+ in: number;
13
+ out: number;
14
+ cacheRead?: number;
15
+ cacheWrite?: number;
16
+ }
17
+
18
+ const PRICES: Record<string, Price> = (() => {
19
+ try {
20
+ const raw = readFileSync(fileURLToPath(new URL("./prices.json", import.meta.url)), "utf-8");
21
+ return (JSON.parse(raw) as { prices?: Record<string, Price> }).prices ?? {};
22
+ } catch {
23
+ return {};
24
+ }
25
+ })();
26
+
27
+ /**
28
+ * 把五花八门的 model 标识归一到价格表的 key:精确命中优先,再退而去掉 provider 前缀
29
+ * (`anthropic/claude-…` → `claude-…`)和末尾日期版本(`…-4-5-20251001` → `…-4-5`)。
30
+ */
31
+ function lookup(model: string): Price | undefined {
32
+ if (PRICES[model]) return PRICES[model];
33
+ const bare = model.includes("/") ? model.slice(model.lastIndexOf("/") + 1) : model;
34
+ if (PRICES[bare]) return PRICES[bare];
35
+ const undated = bare.replace(/-\d{8}$/, "").replace(/-\d{4}-\d{2}-\d{2}$/, "");
36
+ if (PRICES[undated]) return PRICES[undated];
37
+ return undefined;
38
+ }
39
+
40
+ /**
41
+ * 按 token 桶 × 单价估算一次运行的美元成本。cache 桶缺专门单价时退回 input 价
42
+ * (cache token 本质也是 input)。无 model / 查不到价 / 零用量 → undefined。
43
+ */
44
+ export function estimateCost(model: string | undefined, usage: Usage): number | undefined {
45
+ if (!model) return undefined;
46
+ const p = lookup(model);
47
+ if (!p) return undefined;
48
+ const bucket = (tokens: number | undefined, price: number | undefined, fallback: number): number =>
49
+ tokens ? tokens * (price ?? fallback) : 0;
50
+ const usd =
51
+ (bucket(usage.inputTokens, p.in, p.in) +
52
+ bucket(usage.outputTokens, p.out, p.out) +
53
+ bucket(usage.cacheReadTokens, p.cacheRead, p.in) +
54
+ bucket(usage.cacheWriteTokens, p.cacheWrite, p.in)) /
55
+ 1e6;
56
+ return usd > 0 ? usd : undefined;
57
+ }
@@ -0,0 +1,304 @@
1
+ // 从归一化的 StreamEvent[] 折叠出结构化事实。
2
+ // - deriveRunFacts:断言层吃的 DerivedFacts(按 callId 把 called+result 折成 ToolCall);
3
+ // - buildO11ySummary:给人 / EVAL.ts 看的 o11y 摘要(注入 __niceeval__/results.json)。
4
+ // 一旦事件流归一好了,这两个折叠对所有 agent 通用。
5
+
6
+ import type {
7
+ StreamEvent,
8
+ DerivedFacts,
9
+ ToolCall,
10
+ SubagentCall,
11
+ InputRequest,
12
+ O11ySummary,
13
+ TraceSpan,
14
+ Usage,
15
+ ToolName,
16
+ JsonValue,
17
+ } from "../types.ts";
18
+
19
+ // ───────────────────────── 小工具 ─────────────────────────
20
+
21
+ /** 把 JsonValue 当对象取字段;非对象返回 undefined。 */
22
+ function field(input: JsonValue | undefined, key: string): JsonValue | undefined {
23
+ if (input && typeof input === "object" && !Array.isArray(input)) {
24
+ return (input as Record<string, JsonValue>)[key];
25
+ }
26
+ return undefined;
27
+ }
28
+
29
+ /** 按候选 key 顺序取第一个字符串。 */
30
+ function pickString(input: JsonValue | undefined, keys: string[]): string | undefined {
31
+ for (const k of keys) {
32
+ const v = field(input, k);
33
+ if (typeof v === "string" && v) return v;
34
+ }
35
+ return undefined;
36
+ }
37
+
38
+ /** command 可能是 string 或 string[],归一成一行命令文本。 */
39
+ function pickCommand(input: JsonValue | undefined): string | undefined {
40
+ const cmd = field(input, "command") ?? field(input, "cmd");
41
+ if (typeof cmd === "string") return cmd;
42
+ if (Array.isArray(cmd)) return cmd.filter((p) => typeof p === "string").join(" ");
43
+ const program = field(input, "program");
44
+ const args = field(input, "args");
45
+ if (typeof program === "string" && Array.isArray(args)) {
46
+ return `${program} ${args.filter((p) => typeof p === "string").join(" ")}`;
47
+ }
48
+ return undefined;
49
+ }
50
+
51
+ /** 从工具结果 output 里抠 exit_code(codex shell 结果常嵌在 output / metadata 里)。 */
52
+ function pickExitCode(output: JsonValue | undefined): number | undefined {
53
+ const direct = field(output, "exit_code") ?? field(output, "exitCode");
54
+ if (typeof direct === "number") return direct;
55
+ const meta = field(output, "metadata");
56
+ const nested = field(meta, "exit_code") ?? field(meta, "exitCode");
57
+ if (typeof nested === "number") return nested;
58
+ return undefined;
59
+ }
60
+
61
+ // ───────────────────────── deriveRunFacts ─────────────────────────
62
+
63
+ export function deriveRunFacts(events: readonly StreamEvent[]): DerivedFacts {
64
+ const toolCallMap = new Map<string, ToolCall>();
65
+ const toolCallOrder: string[] = [];
66
+ const subagentMap = new Map<string, SubagentCall>();
67
+ const subagentOrder: string[] = [];
68
+ const inputRequests: InputRequest[] = [];
69
+ let messageCount = 0;
70
+ let compactions = 0;
71
+
72
+ for (const ev of events) {
73
+ switch (ev.type) {
74
+ case "message":
75
+ messageCount += 1;
76
+ break;
77
+
78
+ case "action.called": {
79
+ if (!toolCallMap.has(ev.callId)) toolCallOrder.push(ev.callId);
80
+ toolCallMap.set(ev.callId, {
81
+ callId: ev.callId,
82
+ name: ev.tool ?? "unknown",
83
+ originalName: ev.name,
84
+ input: ev.input,
85
+ status: "completed",
86
+ });
87
+ break;
88
+ }
89
+
90
+ case "action.result": {
91
+ const existing = toolCallMap.get(ev.callId);
92
+ if (existing) {
93
+ existing.output = ev.output;
94
+ existing.status = ev.status;
95
+ } else {
96
+ // 只有结果、没配上调用:补一条占位 ToolCall。
97
+ toolCallOrder.push(ev.callId);
98
+ toolCallMap.set(ev.callId, {
99
+ callId: ev.callId,
100
+ name: "unknown",
101
+ input: null,
102
+ output: ev.output,
103
+ status: ev.status,
104
+ });
105
+ }
106
+ break;
107
+ }
108
+
109
+ case "subagent.called": {
110
+ if (!subagentMap.has(ev.callId)) subagentOrder.push(ev.callId);
111
+ subagentMap.set(ev.callId, {
112
+ callId: ev.callId,
113
+ name: ev.name,
114
+ remoteUrl: ev.remoteUrl,
115
+ status: "completed",
116
+ });
117
+ break;
118
+ }
119
+
120
+ case "subagent.completed": {
121
+ const existing = subagentMap.get(ev.callId);
122
+ if (existing) {
123
+ existing.output = ev.output;
124
+ existing.status = ev.status;
125
+ } else {
126
+ subagentOrder.push(ev.callId);
127
+ subagentMap.set(ev.callId, {
128
+ callId: ev.callId,
129
+ name: "unknown",
130
+ output: ev.output,
131
+ status: ev.status,
132
+ });
133
+ }
134
+ break;
135
+ }
136
+
137
+ case "input.requested":
138
+ inputRequests.push(ev.request);
139
+ break;
140
+
141
+ case "compaction":
142
+ compactions += 1;
143
+ break;
144
+
145
+ default:
146
+ break;
147
+ }
148
+ }
149
+
150
+ // parked:最后一条「有意义」的事件是 input.requested(忽略 thinking / compaction 这类尾随噪声)。
151
+ let parked = false;
152
+ for (let i = events.length - 1; i >= 0; i--) {
153
+ const t = events[i].type;
154
+ if (t === "thinking" || t === "compaction") continue;
155
+ parked = t === "input.requested";
156
+ break;
157
+ }
158
+
159
+ return {
160
+ toolCalls: toolCallOrder.map((id) => toolCallMap.get(id)!),
161
+ subagentCalls: subagentOrder.map((id) => subagentMap.get(id)!),
162
+ inputRequests,
163
+ parked,
164
+ messageCount,
165
+ compactions,
166
+ };
167
+ }
168
+
169
+ // ───────────────────────── extractUsageFromSpans ─────────────────────────
170
+
171
+ /**
172
+ * adapter 未报 usage 时的兜底:从 OTLP span 属性里提取 token 用量。
173
+ * 按 OpenTelemetry GenAI 语义约定累加所有模型调用 span 的用量字段。
174
+ * 返回 undefined 表示 span 里也没有用量信息。
175
+ */
176
+ export function extractUsageFromSpans(spans: readonly TraceSpan[]): Usage | undefined {
177
+ let inputTokens = 0;
178
+ let outputTokens = 0;
179
+ let cacheReadTokens = 0;
180
+
181
+ for (const span of spans) {
182
+ const a = span.attributes ?? {};
183
+ // OpenTelemetry GenAI 语义约定(新旧两套 key 都认)
184
+ inputTokens += numAttr(a, "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens");
185
+ outputTokens += numAttr(a, "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens");
186
+ cacheReadTokens += numAttr(a, "gen_ai.usage.cache_read_input_tokens");
187
+ }
188
+
189
+ if (inputTokens === 0 && outputTokens === 0) return undefined;
190
+ const u: Usage = { inputTokens, outputTokens };
191
+ if (cacheReadTokens > 0) u.cacheReadTokens = cacheReadTokens;
192
+ return u;
193
+ }
194
+
195
+ function numAttr(attrs: Record<string, JsonValue>, ...keys: string[]): number {
196
+ for (const k of keys) {
197
+ const v = attrs[k];
198
+ if (typeof v === "number" && v > 0) return v;
199
+ }
200
+ return 0;
201
+ }
202
+
203
+ // ───────────────────────── buildO11ySummary ─────────────────────────
204
+
205
+ export function buildO11ySummary(
206
+ events: readonly StreamEvent[],
207
+ usage: Usage,
208
+ durationMs: number,
209
+ ): O11ySummary {
210
+ const toolCalls: Record<string, number> = {};
211
+ let totalToolCalls = 0;
212
+ const filesRead = new Set<string>();
213
+ const filesModified = new Set<string>();
214
+ const shellCommands: { command: string; exitCode?: number; success?: boolean }[] = [];
215
+ const webFetches: { url: string; status?: number; success?: boolean }[] = [];
216
+ const errors: string[] = [];
217
+ let thinkingBlocks = 0;
218
+ let compactions = 0;
219
+ let totalTurns = 0;
220
+
221
+ // 先把结果按 callId 建索引,供 shell / web 回填成败。
222
+ const resultByCallId = new Map<string, Extract<StreamEvent, { type: "action.result" }>>();
223
+ for (const ev of events) {
224
+ if (ev.type === "action.result") resultByCallId.set(ev.callId, ev);
225
+ }
226
+
227
+ for (const ev of events) {
228
+ switch (ev.type) {
229
+ case "message":
230
+ if (ev.role === "assistant") totalTurns += 1;
231
+ break;
232
+
233
+ case "thinking":
234
+ thinkingBlocks += 1;
235
+ break;
236
+
237
+ case "compaction":
238
+ compactions += 1;
239
+ break;
240
+
241
+ case "error":
242
+ if (ev.message) errors.push(ev.message);
243
+ break;
244
+
245
+ case "action.called": {
246
+ const canonical: ToolName | string = ev.tool ?? "unknown";
247
+ toolCalls[canonical] = (toolCalls[canonical] ?? 0) + 1;
248
+ totalToolCalls += 1;
249
+
250
+ const input = ev.input;
251
+ const result = resultByCallId.get(ev.callId);
252
+
253
+ if (ev.tool === "file_read") {
254
+ const path = pickString(input, ["path", "file", "file_path", "filename"]);
255
+ if (path) filesRead.add(path);
256
+ } else if (ev.tool === "file_write" || ev.tool === "file_edit") {
257
+ const path = pickString(input, ["path", "file", "file_path", "filename"]);
258
+ if (path) filesModified.add(path);
259
+ } else if (ev.tool === "shell") {
260
+ const command = pickCommand(input);
261
+ if (command) {
262
+ const entry: { command: string; exitCode?: number; success?: boolean } = { command };
263
+ if (result) {
264
+ entry.success = result.status === "completed";
265
+ const exit = pickExitCode(result.output);
266
+ if (exit !== undefined) entry.exitCode = exit;
267
+ }
268
+ shellCommands.push(entry);
269
+ }
270
+ } else if (ev.tool === "web_fetch") {
271
+ const url = pickString(input, ["url", "uri", "endpoint", "href"]);
272
+ if (url) {
273
+ const entry: { url: string; status?: number; success?: boolean } = { url };
274
+ if (result) {
275
+ entry.success = result.status === "completed";
276
+ const status = field(result.output, "status");
277
+ if (typeof status === "number") entry.status = status;
278
+ }
279
+ webFetches.push(entry);
280
+ }
281
+ }
282
+ break;
283
+ }
284
+
285
+ default:
286
+ break;
287
+ }
288
+ }
289
+
290
+ return {
291
+ totalTurns,
292
+ toolCalls,
293
+ totalToolCalls,
294
+ filesRead: Array.from(filesRead),
295
+ filesModified: Array.from(filesModified),
296
+ shellCommands,
297
+ webFetches,
298
+ errors,
299
+ thinkingBlocks,
300
+ compactions,
301
+ durationMs,
302
+ usage,
303
+ };
304
+ }
@@ -0,0 +1,112 @@
1
+ // Canonical trace schema = OpenTelemetry GenAI 语义约定。
2
+ //
3
+ // 「定义我们的 OTel」不是发明 niceeval 私有 schema —— 我们控制不了 agent 的 instrumentation,
4
+ // 造私有约定也强迫不了 codex(Rust)/ claude 原生发它。所以 canonical 目标就是 OTel 官方的
5
+ // GenAI semconv(https://opentelemetry.io/docs/specs/semconv/gen-ai/):一个公认标准,
6
+ // codex 已部分遵循、bub 插件可配置直接发。
7
+ //
8
+ // 核心是 gen_ai.operation.name 把 span 分成几类语义角色(SpanKind),view/select 只认 SpanKind、
9
+ // 不读原生 span 名 —— 于是不同 agent 的瀑布图天然对齐、可叠加对比。每个 agent 一个薄 mapper
10
+ // (otlp/mappers/<agent>.ts)把原生 span 归一到这套;本文件提供它们共用的常量 / 助手 / 通用兜底。
11
+
12
+ import type { JsonValue, SpanKind, TraceSpan } from "../../types.ts";
13
+
14
+ // ───────────────────────── GenAI semconv 键 / 操作名 ─────────────────────────
15
+
16
+ /** gen_ai.operation.name —— canonical 的核心:span 干的是哪类活。 */
17
+ export const GENAI_OP = "gen_ai.operation.name";
18
+
19
+ export const OP_CHAT = "chat";
20
+ export const OP_TEXT_COMPLETION = "text_completion";
21
+ export const OP_EMBEDDINGS = "embeddings";
22
+ export const OP_EXECUTE_TOOL = "execute_tool";
23
+ export const OP_INVOKE_AGENT = "invoke_agent";
24
+ export const OP_CREATE_AGENT = "create_agent";
25
+
26
+ /** gen_ai.operation.name → niceeval SpanKind(view 着色 / 分组的唯一依据)。 */
27
+ export function kindFromOperation(op: string | undefined): SpanKind {
28
+ switch (op) {
29
+ case OP_CHAT:
30
+ case OP_TEXT_COMPLETION:
31
+ case OP_EMBEDDINGS:
32
+ return "model";
33
+ case OP_EXECUTE_TOOL:
34
+ return "tool";
35
+ case OP_INVOKE_AGENT:
36
+ case OP_CREATE_AGENT:
37
+ return "agent";
38
+ default:
39
+ return "other";
40
+ }
41
+ }
42
+
43
+ // ───────────────────────── 给 span 打 canonical 标 ─────────────────────────
44
+
45
+ /** mapper 每条 span 归一后的判定:认出的标准操作(可空)+ 语义角色。 */
46
+ export interface SpanTag {
47
+ /** 标准 GenAI 操作名;turn 骨架这类 GenAI 没对应操作的留空,只给 kind。 */
48
+ op?: string;
49
+ /** 语义角色。给了 op 时一般 = kindFromOperation(op);turn 骨架直接给 "turn"。 */
50
+ kind: SpanKind;
51
+ }
52
+
53
+ /**
54
+ * 把判定写回 span:有标准 op 时写入 gen_ai.operation.name(供下游如 extractUsageFromSpans 用),
55
+ * 并定 kind。**raw name / 既有 attributes 一律保留**,只增不改,供 view 下钻。
56
+ */
57
+ export function tagSpan(span: TraceSpan, tag: SpanTag): TraceSpan {
58
+ if (!tag.op) return { ...span, kind: tag.kind };
59
+ const attributes: Record<string, JsonValue> = { ...span.attributes, [GENAI_OP]: tag.op };
60
+ return { ...span, kind: tag.kind, attributes };
61
+ }
62
+
63
+ // ───────────────────────── 通用兜底 mapper(heuristic) ─────────────────────────
64
+ //
65
+ // 给【没有专属 mapper 的 agent】兜底:按 GenAI 语义信号 + 常见动词名猜操作。
66
+ // 有专属 mapper 的(codex/bub)在自己文件里编码本 agent 的名字约定,只在不认得时回落到这里。
67
+
68
+ /**
69
+ * 按通用信号判定一条 span 的 canonical 角色:
70
+ * · 已带 gen_ai.operation.name —— 直接信任(上游已按标准发);
71
+ * · 工具执行 —— 带 tool_name 属性,或名字是执行/路由动词;
72
+ * · 模型调用 —— 带 gen_ai.* / 到模型的 HTTP / 采样·流式名;
73
+ * · 回合·会话骨架 —— turn.id 属性,或回合/会话动词名;
74
+ * · 都不沾 —— "other"(view 折叠)。
75
+ */
76
+ export function heuristicTag(span: TraceSpan): SpanTag {
77
+ const ln = span.name.toLowerCase();
78
+ const a = span.attributes ?? {};
79
+ const keys = Object.keys(a);
80
+
81
+ // 上游已按 GenAI 标准发了操作名 —— 直接信任。
82
+ const existing = a[GENAI_OP];
83
+ if (typeof existing === "string") return { op: existing, kind: kindFromOperation(existing) };
84
+
85
+ // 工具执行。
86
+ if ("tool_name" in a) return { op: OP_EXECUTE_TOOL, kind: "tool" };
87
+ if (/(^|[._])(exec_command|apply_patch|write_stdin|execute_tool|run_command|handle_tool_call|dispatch_tool_call)/.test(ln)) {
88
+ return { op: OP_EXECUTE_TOOL, kind: "tool" };
89
+ }
90
+
91
+ // 模型调用。
92
+ if (keys.some((k) => k.startsWith("gen_ai.request") || k.startsWith("gen_ai.response"))) return { op: OP_CHAT, kind: "model" };
93
+ if ("wire_api" in a || ("http.method" in a && "api.path" in a)) return { op: OP_CHAT, kind: "model" };
94
+ if (/(^|[._])(run_sampling_request|try_run_sampling_request|stream_responses|receiving_stream|model_client|chat|completion)(\b|_|$)/.test(ln)) {
95
+ return { op: OP_CHAT, kind: "model" };
96
+ }
97
+
98
+ // 子 agent 调用。
99
+ if (/(^|[._])create_agent(\b|_|$)/.test(ln)) return { op: OP_CREATE_AGENT, kind: "agent" };
100
+ if (/(^|[._])invoke_agent(\b|_|$)/.test(ln)) return { op: OP_INVOKE_AGENT, kind: "agent" };
101
+
102
+ // 回合 / 会话骨架(GenAI 无对应操作 → 只给 kind)。
103
+ if ("turn.id" in a || keys.some((k) => k.startsWith("codex.turn"))) return { kind: "turn" };
104
+ if (/^(codex\.exec|session_loop|run_turn)$/.test(ln) || /session_task\.turn|agent\.step/.test(ln)) return { kind: "turn" };
105
+
106
+ return { kind: "other" };
107
+ }
108
+
109
+ /** 通用 mapper:逐 span 套 heuristicTag。无专属 mapper 的 agent 走它。 */
110
+ export function mapGenericSpans(spans: TraceSpan[]): TraceSpan[] {
111
+ return spans.map((s) => tagSpan(s, heuristicTag(s)));
112
+ }
@@ -0,0 +1,30 @@
1
+ // bub 的 span mapper:原生 OTLP span → canonical GenAI semconv。
2
+ //
3
+ // bub-tapestore-otel 插件把 tape 装饰成 span,名字本就接近 GenAI semconv
4
+ // (invoke_agent / agent.step / chat / execute_tool,见 agents/bub.ts 注释),
5
+ // 所以这个 mapper 近乎透传 —— 只把 bub 自己的 agent.step(回合步)归到 "turn",
6
+ // 其余直接按名映到标准操作;不认得的回落通用 heuristic。
7
+
8
+ import type { TraceSpan } from "../../../types.ts";
9
+ import { OP_CHAT, OP_CREATE_AGENT, OP_EXECUTE_TOOL, OP_INVOKE_AGENT, heuristicTag, tagSpan, type SpanTag } from "../canonical.ts";
10
+
11
+ function bubTag(span: TraceSpan): SpanTag {
12
+ switch (span.name.toLowerCase()) {
13
+ case "execute_tool":
14
+ return { op: OP_EXECUTE_TOOL, kind: "tool" };
15
+ case "chat":
16
+ return { op: OP_CHAT, kind: "model" };
17
+ case "invoke_agent":
18
+ return { op: OP_INVOKE_AGENT, kind: "agent" };
19
+ case "create_agent":
20
+ return { op: OP_CREATE_AGENT, kind: "agent" };
21
+ case "agent.step": // bub 的回合步:GenAI 无对应操作 → 只给 kind。
22
+ return { kind: "turn" };
23
+ default:
24
+ return heuristicTag(span);
25
+ }
26
+ }
27
+
28
+ export function mapBubSpans(spans: TraceSpan[]): TraceSpan[] {
29
+ return spans.map((s) => tagSpan(s, bubTag(s)));
30
+ }
@@ -0,0 +1,38 @@
1
+ // codex 的 span mapper:原生 OTLP span → canonical GenAI semconv。
2
+ //
3
+ // codex(Rust)的 OTLP 把内部 tracing 全导出来,span 名是 codex 自家约定(codex.exec /
4
+ // run_sampling_request / exec_command …),不是 GenAI semconv。这里编码 codex 的名字约定,
5
+ // 把它们归一到 gen_ai.operation.name + SpanKind;不认得的回落通用 heuristic。
6
+ // 理想态:codex 在 config.toml 里直接发 GenAI 语义 → 这个 mapper 就近乎透传(走 heuristic 的
7
+ // 「已带 gen_ai.operation.name → 信任」分支)。在那之前,由本文件兜。
8
+
9
+ import type { TraceSpan } from "../../../types.ts";
10
+ import { OP_CHAT, OP_EXECUTE_TOOL, heuristicTag, tagSpan, type SpanTag } from "../canonical.ts";
11
+
12
+ function codexTag(span: TraceSpan): SpanTag {
13
+ const ln = span.name.toLowerCase();
14
+ const a = span.attributes ?? {};
15
+
16
+ // 工具执行:带 tool_name/call_id,或 codex 的执行/补丁/路由 span 名。
17
+ if ("tool_name" in a) return { op: OP_EXECUTE_TOOL, kind: "tool" };
18
+ if (/(^|[._])(exec_command|apply_patch|write_stdin|handle_tool_call|dispatch_tool_call)/.test(ln)) {
19
+ return { op: OP_EXECUTE_TOOL, kind: "tool" };
20
+ }
21
+
22
+ // 模型调用:codex 的采样 / 流式 span,或到模型后端的 HTTP(wire_api / responses)。
23
+ if ("wire_api" in a) return { op: OP_CHAT, kind: "model" };
24
+ if (/(^|[._])(run_sampling_request|try_run_sampling_request|stream_responses|receiving_stream)(\b|_|$)/.test(ln)) {
25
+ return { op: OP_CHAT, kind: "model" };
26
+ }
27
+
28
+ // 回合 / 会话骨架:codex.exec(一次 exec 调用)、run_turn、turn.id 属性。GenAI 无对应操作 → 只给 kind。
29
+ if ("turn.id" in a || Object.keys(a).some((k) => k.startsWith("codex.turn"))) return { kind: "turn" };
30
+ if (/^(codex\.exec|run_turn|session_loop)$/.test(ln) || /session_task\.turn/.test(ln)) return { kind: "turn" };
31
+
32
+ // codex 没专门命名的(handle_responses / append_items 等内部噪声)交给通用兜底 → 多半落 "other"。
33
+ return heuristicTag(span);
34
+ }
35
+
36
+ export function mapCodexSpans(spans: TraceSpan[]): TraceSpan[] {
37
+ return spans.map((s) => tagSpan(s, codexTag(s)));
38
+ }
@@ -0,0 +1,25 @@
1
+ // span mapper 入口:把各 agent 的原生 OTLP span 归一到 canonical GenAI semconv。
2
+ // 和 transcript 解析器(o11y/parsers/index.ts)同一个分派模式 —— 按 agent 名包含匹配,
3
+ // 认不出来走通用 heuristic 兜底。每个 mapper 是纯函数(不碰沙箱),住 core o11y、可独立单测。
4
+ //
5
+ // 职责边界:本文件只管「发回来的 span 怎么读」;「沙箱里怎么让 agent 把 OTLP 发出来」
6
+ //(写 config.toml / 注入 OTEL_* env)是 adapter 侧的导出配置,与此分开。
7
+
8
+ import type { TraceSpan } from "../../../types.ts";
9
+ import { mapGenericSpans } from "../canonical.ts";
10
+ import { mapCodexSpans } from "./codex.ts";
11
+ import { mapBubSpans } from "./bub.ts";
12
+
13
+ export { mapCodexSpans, mapBubSpans };
14
+
15
+ /**
16
+ * 按 agent 名分派到对应 span mapper;认不出来的走通用 heuristic 兜底。
17
+ * agent 名做包含匹配,兼容 "vercel-ai-gateway/codex" 之类变体(同 parseTranscript)。
18
+ */
19
+ export function mapSpansToCanonical(spans: TraceSpan[], agent: string): TraceSpan[] {
20
+ const a = (agent || "").toLowerCase();
21
+ if (a.includes("codex")) return mapCodexSpans(spans);
22
+ if (a.includes("bub")) return mapBubSpans(spans);
23
+ // claude-code 不发 OTLP(无 span 到这);其余未知 agent 走通用兜底。
24
+ return mapGenericSpans(spans);
25
+ }