niceeval 0.1.1 → 0.2.0

package/src/o11y/parsers/claude-code.ts

@@ -7,61 +7,37 @@
 
 import type { StreamEvent, Usage, ToolName, JsonValue } from "../../types.ts";
 import type { ParsedTranscript } from "./index.ts";
+import { normalizeToolName as normalizeShared } from "../tool-names.ts";
 
 // ───────────────────────── 工具名归一 ─────────────────────────
 
-function normalizeToolName(name: string): ToolName {
-  const toolMap: Record<string, ToolName> = {
-    // 文件
-    Read: "file_read",
-    read_file: "file_read",
-    ReadFile: "file_read",
-    Write: "file_write",
-    write_file: "file_write",
-    WriteFile: "file_write",
-    write_to_file: "file_write",
-    create_file: "file_write",
-    Edit: "file_edit",
-    MultiEdit: "file_edit",
-    edit_file: "file_edit",
-    EditFile: "file_edit",
-    str_replace_editor: "file_edit",
-    StrReplace: "file_edit",
-    NotebookEdit: "file_edit",
-
-    // shell
-    Bash: "shell",
-    bash: "shell",
-    BashOutput: "shell",
-    Shell: "shell",
-    shell: "shell",
-    execute_command: "shell",
-    run_command: "shell",
-
-    // web
-    WebFetch: "web_fetch",
-    web_fetch: "web_fetch",
-    fetch_url: "web_fetch",
-    mcp__fetch__fetch: "web_fetch",
-    WebSearch: "web_search",
-    web_search: "web_search",
+/**
+ * Claude Code 特有别名(键小写;PascalCase 原名靠共享层的 toLowerCase 兜住);通用别名走基表。
+ * 裸名 read/write/edit/webfetch/websearch/task 必须在这里:它们是 Claude Code 的核心工具
+ * (Read/Write/Edit/WebFetch/WebSearch/Task),基表刻意不收裸动词。
+ */
+export const CLAUDE_TOOL_ALIASES: Record<string, ToolName> = {
+  read: "file_read",
+  readfile: "file_read",
+  write: "file_write",
+  writefile: "file_write",
+  write_to_file: "file_write",
+  edit: "file_edit",
+  multiedit: "file_edit",
+  editfile: "file_edit",
+  strreplace: "file_edit",
+  notebookedit: "file_edit",
+  bashoutput: "shell",
+  webfetch: "web_fetch",
+  mcp__fetch__fetch: "web_fetch",
+  websearch: "web_search",
+  ls: "list_dir",
+  listdir: "list_dir",
+  task: "agent_task",
+};
 
-    // 检索 / 导航
-    Glob: "glob",
-    glob: "glob",
-    list_files: "glob",
-    Grep: "grep",
-    grep: "grep",
-    search_files: "grep",
-    LS: "list_dir",
-    list_dir: "list_dir",
-    ListDir: "list_dir",
-
-    // 子 agent
-    Task: "agent_task",
-    task: "agent_task",
-  };
-  return toolMap[name] || "unknown";
+function normalizeToolName(name: string): ToolName {
+  return normalizeShared(name, CLAUDE_TOOL_ALIASES);
 }
 
 // ───────────────────────── 小工具 ─────────────────────────

package/src/o11y/parsers/codex.ts

@@ -6,53 +6,19 @@
 
 import type { StreamEvent, Usage, ToolName, JsonValue } from "../../types.ts";
 import type { ParsedTranscript } from "./index.ts";
+import { GENERIC_VERB_ALIASES, normalizeToolName as normalizeShared } from "../tool-names.ts";
 
 // ───────────────────────── 工具名归一 ─────────────────────────
 
-/** Codex 工具名 → 规范 ToolName。 */
+/** Codex 特有别名 + 裸动词(codex transcript 的工具名由 CLI 控制,裸动词安全);通用别名走基表。 */
+export const CODEX_TOOL_ALIASES: Record<string, ToolName> = {
+  ...GENERIC_VERB_ALIASES,
+  file_change: "file_edit",
+  update_plan: "agent_task",
+};
+
 function normalizeToolName(name: string): ToolName {
-  const toolMap: Record<string, ToolName> = {
-    // 文件
-    read_file: "file_read",
-    write_file: "file_write",
-    create_file: "file_write",
-    delete_file: "file_write",
-    edit_file: "file_edit",
-    patch_file: "file_edit",
-    apply_patch: "file_edit",
-    file_change: "file_edit",
-    update_plan: "agent_task",
-
-    // shell
-    shell: "shell",
-    bash: "shell",
-    exec: "shell",
-    execute: "shell",
-    run: "shell",
-    terminal: "shell",
-    command_execution: "shell",
-    local_shell: "shell",
-
-    // web
-    fetch: "web_fetch",
-    http_request: "web_fetch",
-    curl: "web_fetch",
-    web_fetch: "web_fetch",
-    web_search: "web_search",
-    search: "web_search",
-
-    // 检索 / 导航
-    glob: "glob",
-    find_files: "glob",
-    list_files: "glob",
-    grep: "grep",
-    search_files: "grep",
-    ripgrep: "grep",
-    ls: "list_dir",
-    list_directory: "list_dir",
-    dir: "list_dir",
-  };
-  return toolMap[name.toLowerCase()] || "unknown";
+  return normalizeShared(name, CODEX_TOOL_ALIASES);
 }
 
 // ───────────────────────── 小工具 ─────────────────────────
@@ -140,13 +106,17 @@ function pickUsage(data: unknown): unknown {
 
 // ───────────────────────── compaction 标记 ─────────────────────────
 
-/** 仅在出现可信压缩标记时才认(保守):type / item.type 含 compact / summariz / context_truncat。 */
+/**
+ * 仅在出现可信压缩标记时才认(保守):type / item.type / subtype 含 compact 或 context_truncat。
+ * 不再匹配 "summariz"——任何名字带 summarize 的工具 / item(如 summarization_tool)都会被
+ * 误判成压缩边界并整条吞掉,污染计数且丢事件。命中即吞整条事件,匹配面必须窄。
+ */
 function isCompactionMarker(eventType: unknown, data: unknown): boolean {
   const candidates = [eventType, get(get(data, "item"), "type"), get(data, "subtype")];
   for (const c of candidates) {
     if (typeof c !== "string") continue;
     const s = c.toLowerCase();
-    if (s.includes("compact") || s.includes("summariz") || s.includes("context_truncat")) return true;
+    if (s.includes("compact") || s.includes("context_truncat")) return true;
   }
   return false;
 }
@@ -447,7 +417,11 @@ export function parseCodexTranscript(raw: string | undefined): ParsedTranscript
   return { events, usage, compactions, parseSuccess };
 }
 
-/** 解读一条工具结果输出:抠 exit_code / error 推断成败,返回归一化 output + status。 */
+/**
+ * 解读一条工具结果输出:显式 success/error → status 字符串 → exit_code,层层找成败证据。
+ * 都没有才默认 completed —— 这是乐观兜底:codex 不带任何成败字段的结果会被记成成功,
+ * noFailedActions() 这类负断言可能因此假通过。所以证据检查要尽量全,别轻易走到兜底。
+ */
 function interpretOutput(
   rawOut: unknown,
   data: unknown,
@@ -457,6 +431,15 @@ function interpretOutput(
   if (explicitSuccess === false) return { output: (rawOut ?? null) as JsonValue, status: "failed" };
   if (get(data, "error")) return { output: (rawOut ?? null) as JsonValue, status: "failed" };
 
+  // 事件自带 status 字符串(item/data 两层都看)。
+  const statusStr = get(data, "status") ?? get(get(data, "item"), "status");
+  if (statusStr === "failed" || statusStr === "error") {
+    return { output: (rawOut ?? null) as JsonValue, status: "failed" };
+  }
+  if (statusStr === "rejected" || statusStr === "declined") {
+    return { output: (rawOut ?? null) as JsonValue, status: "rejected" };
+  }
+
   // 字符串输出尝试解析 exit_code(codex shell 结果常见)。
   let parsed: unknown = rawOut;
   if (typeof rawOut === "string") {

package/src/o11y/parsers/index.ts

@@ -1,5 +1,9 @@
 // o11y 解析器入口:把各 agent 的原始 transcript 归一成 niceeval StreamEvent[]。
-// 每个解析器都返回同一个 ParsedTranscript 形状,parseTranscript 按 agent 名分派。
+// 每个解析器都返回同一个 ParsedTranscript 形状。
+//
+// 这里没有「按 agent 名分派」的入口:解析器由 adapter 直连(shared.parseCodex /
+// parseClaudeCode / parseBub),core 不按名字分支。接新 agent = 写新解析器 + adapter
+// 里直接调,不改本文件的任何分派逻辑。
 
 import type { StreamEvent, Usage } from "../../types.ts";
 import { parseCodexTranscript, parseCodex } from "./codex.ts";
@@ -22,16 +26,3 @@ export {
   parseBubTranscript,
   parseBub,
 };
-
-/**
- * 按 agent 名分派到对应解析器;认不出来的走通用(bub)兜底。
- * agent 名做包含匹配,兼容 "vercel-ai-gateway/codex"、"claude-code-1m" 之类的变体。
- */
-export function parseTranscript(raw: string | undefined, agent: string): ParsedTranscript {
-  const a = (agent || "").toLowerCase();
-  if (a.includes("codex")) return parseCodexTranscript(raw);
-  if (a.includes("claude")) return parseClaudeCodeTranscript(raw);
-  if (a.includes("bub")) return parseBubTranscript(raw);
-  // 默认:通用 JSONL 兜底解析器。
-  return parseBubTranscript(raw);
-}

package/src/o11y/tool-names.test.ts

@@ -0,0 +1,61 @@
+// 工具名归一的回归锁:canonical ToolName 是跨 agent 断言(calledTool("file_read"))
+// 的命中基础。这里逐 agent 锁死关键映射,防止基表 / per-agent 差异表重构时悄悄漂移
+// (曾发生:Claude Code 的裸名 Read/Write/Edit 在一次表合并中归一成了 "unknown")。
+
+import { describe, it, expect } from "vitest";
+import { normalizeToolName } from "./tool-names.ts";
+import { CLAUDE_TOOL_ALIASES } from "./parsers/claude-code.ts";
+import { CODEX_TOOL_ALIASES } from "./parsers/codex.ts";
+import { BUB_TOOL_ALIASES } from "./parsers/bub.ts";
+
+describe("normalizeToolName", () => {
+  it("claude-code:PascalCase 核心工具全部命中", () => {
+    const n = (name: string) => normalizeToolName(name, CLAUDE_TOOL_ALIASES);
+    expect(n("Read")).toBe("file_read");
+    expect(n("Write")).toBe("file_write");
+    expect(n("Edit")).toBe("file_edit");
+    expect(n("MultiEdit")).toBe("file_edit");
+    expect(n("NotebookEdit")).toBe("file_edit");
+    expect(n("Bash")).toBe("shell");
+    expect(n("BashOutput")).toBe("shell");
+    expect(n("WebFetch")).toBe("web_fetch");
+    expect(n("WebSearch")).toBe("web_search");
+    expect(n("Glob")).toBe("glob");
+    expect(n("Grep")).toBe("grep");
+    expect(n("LS")).toBe("list_dir");
+    expect(n("Task")).toBe("agent_task");
+  });
+
+  it("codex:裸动词与专属名命中", () => {
+    const n = (name: string) => normalizeToolName(name, CODEX_TOOL_ALIASES);
+    expect(n("exec")).toBe("shell");
+    expect(n("local_shell")).toBe("shell");
+    expect(n("apply_patch")).toBe("file_edit");
+    expect(n("file_change")).toBe("file_edit");
+    expect(n("update_plan")).toBe("agent_task");
+    expect(n("web_search")).toBe("web_search");
+  });
+
+  it("bub:fs.* 命名空间与裸名命中", () => {
+    const n = (name: string) => normalizeToolName(name, BUB_TOOL_ALIASES);
+    expect(n("fs.read")).toBe("file_read");
+    expect(n("fs_write")).toBe("file_write");
+    expect(n("edit")).toBe("file_edit");
+    expect(n("bash")).toBe("shell");
+    expect(n("update_todos")).toBe("agent_task");
+  });
+
+  it("无 overrides(AI SDK 应用):域内可能撞名的裸动词不误归一", () => {
+    // 被测应用完全可能有叫 search / run / fetch / task 的域内工具;
+    // 它们必须落 "unknown",否则 notCalledTool("web_search") 这类断言语义被悄悄改写。
+    expect(normalizeToolName("search")).toBe("unknown");
+    expect(normalizeToolName("run")).toBe("unknown");
+    expect(normalizeToolName("fetch")).toBe("unknown");
+    expect(normalizeToolName("task")).toBe("unknown");
+    expect(normalizeToolName("get_weather")).toBe("unknown");
+    // 但明确的复合名照常命中。
+    expect(normalizeToolName("read_file")).toBe("file_read");
+    expect(normalizeToolName("execute_command")).toBe("shell");
+    expect(normalizeToolName("web_search")).toBe("web_search");
+  });
+});

package/src/o11y/tool-names.ts

@@ -0,0 +1,74 @@
+// 规范工具名归一的唯一出处。canonical ToolName 是跨 agent 断言(calledTool("file_read"))
+// 的命中基础:基表收「跨 agent 一致」的通用别名,各 parser 只声明自己特有的别名叠加在
+// 基表上——新增一个 canonical 名只改这里,不必同步 4 张漂移的表。
+
+import type { ToolName } from "../types.ts";
+
+/**
+ * 通用别名基表(键一律小写;normalizeToolName 匹配前先 toLowerCase)。
+ * 只收「不会撞上被测应用域内工具名」的保守别名(多为带下划线的复合名);
+ * `search`/`run`/`fetch` 这类裸动词在 GENERIC_VERB_ALIASES,由确知自己 transcript
+ * 词汇的 parser 显式 opt-in —— 否则 AI SDK 应用里一个叫 `search` 的商品搜索工具
+ * 会被误归成 web_search,负断言语义被悄悄改写。
+ */
+const BASE_TOOL_ALIASES: Record<string, ToolName> = {
+  // 文件
+  read_file: "file_read",
+  write_file: "file_write",
+  create_file: "file_write",
+  delete_file: "file_write",
+  edit_file: "file_edit",
+  patch_file: "file_edit",
+  apply_patch: "file_edit",
+  str_replace_editor: "file_edit",
+
+  // shell
+  shell: "shell",
+  bash: "shell",
+  command_execution: "shell",
+  local_shell: "shell",
+  execute_command: "shell",
+  run_command: "shell",
+
+  // web
+  web_fetch: "web_fetch",
+  fetch_url: "web_fetch",
+  http_request: "web_fetch",
+  web_search: "web_search",
+
+  // 检索 / 导航
+  glob: "glob",
+  find_files: "glob",
+  list_files: "glob",
+  grep: "grep",
+  search_files: "grep",
+  ripgrep: "grep",
+  list_dir: "list_dir",
+  list_directory: "list_dir",
+};
+
+/**
+ * 裸动词别名:coding-agent CLI 自己的 transcript 词汇(codex/bub/claude 的工具名
+ * 由 CLI 控制,不会和用户域名冲突)。对被测应用(如 AI SDK 的自定义工具)不适用。
+ */
+export const GENERIC_VERB_ALIASES: Record<string, ToolName> = {
+  exec: "shell",
+  execute: "shell",
+  run: "shell",
+  terminal: "shell",
+  fetch: "web_fetch",
+  curl: "web_fetch",
+  search: "web_search",
+  ls: "list_dir",
+  dir: "list_dir",
+  task: "agent_task",
+};
+
+/**
+ * 原始工具名 → 规范 ToolName。先查 agent 特有别名(overrides),再查通用基表,
+ * 都没有则 "unknown"。键匹配不区分大小写(Claude Code 的 PascalCase 名靠这里兜住)。
+ */
+export function normalizeToolName(name: string, overrides?: Record<string, ToolName>): ToolName {
+  const key = name.toLowerCase();
+  return overrides?.[key] ?? BASE_TOOL_ALIASES[key] ?? "unknown";
+}

package/src/o11y/types.ts

@@ -0,0 +1,135 @@
+// o11y 域类型:标准事件流(StreamEvent)、折叠事实(DerivedFacts)、
+// OTLP trace(TraceSpan / SpanKind)与用量 / 摘要。
+
+import type { JsonValue, SourceLoc } from "../shared/types.ts";
+
+/** 一次运行的 token 用量(沙箱型从 transcript 抠,remote 由 send 返回)。 */
+export interface Usage {
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens?: number;
+  cacheWriteTokens?: number;
+  requests?: number;
+  /** 网关实测成本(若 agent 带回)——优先于价格表估算。 */
+  costUSD?: number;
+}
+
+/** 跨 agent 归一化后的规范工具名。 */
+export type ToolName =
+  | "file_read"
+  | "file_write"
+  | "file_edit"
+  | "shell"
+  | "web_fetch"
+  | "web_search"
+  | "glob"
+  | "grep"
+  | "list_dir"
+  | "agent_task"
+  | "unknown";
+
+export interface InputRequest {
+  readonly id?: string;
+  readonly prompt?: string;
+  readonly display?: string;
+  readonly action?: string;
+  readonly input?: JsonValue;
+  readonly options?: readonly { id: string; label?: string }[];
+}
+
+/**
+ * 标准事件流的词汇(对标 docs/agents-and-adapters.md)。adapter 唯一的硬活就是把
+ * 各 agent 五花八门的原始 transcript 映射成 StreamEvent[];映射完,整套断言免费。
+ */
+export type StreamEvent =
+  | { type: "message"; role: "assistant" | "user"; text: string; loc?: SourceLoc }
+  | { type: "action.called"; callId: string; name: string; input: JsonValue; tool?: ToolName }
+  | {
+      type: "action.result";
+      callId: string;
+      output?: JsonValue;
+      status: "completed" | "failed" | "rejected";
+    }
+  | { type: "subagent.called"; callId: string; name: string; remoteUrl?: string }
+  | { type: "subagent.completed"; callId: string; output?: JsonValue; status: "completed" | "failed" }
+  | { type: "input.requested"; request: InputRequest }
+  | { type: "thinking"; text: string }
+  | { type: "compaction"; reason?: string }
+  | { type: "error"; message: string };
+
+/** core 从事件流折叠出的结构化事实(deriveRunFacts)。 */
+export interface ToolCall {
+  callId: string;
+  name: ToolName;
+  originalName?: string;
+  input: JsonValue;
+  output?: JsonValue;
+  status: "completed" | "failed" | "rejected";
+}
+
+export interface SubagentCall {
+  callId: string;
+  name: string;
+  remoteUrl?: string;
+  output?: JsonValue;
+  status: "completed" | "failed";
+}
+
+export interface DerivedFacts {
+  readonly toolCalls: readonly ToolCall[];
+  readonly subagentCalls: readonly SubagentCall[];
+  readonly inputRequests: readonly InputRequest[];
+  readonly parked: boolean;
+  readonly messageCount: number;
+  readonly compactions: number;
+}
+
+/**
+ * span 的【语义角色】,从 OTel GenAI 语义约定的 gen_ai.operation.name 归一而来
+ * (见 o11y/otlp/canonical.ts)。view 据此着色 / 分组 / 跨 agent 对比,**只认这个,
+ * 不读原生 span 名**。未识别的 span 落 "other",view 折叠。
+ */
+export type SpanKind = "turn" | "model" | "tool" | "agent" | "other";
+
+/**
+ * 一条分布式追踪的 span(从 agent 经 OpenTelemetry 导出的 OTLP traces 归一而来)。
+ * 与 StreamEvent 不同:它带【时间】(起止 epoch 毫秒)与【父子】(parentSpanId),
+ * 所以 view 能画成瀑布图。事件流回答「做了什么」,trace 回答「各花了多久、谁套谁」。
+ *
+ * 两层归一:线格式层(OTLP/JSON|protobuf → 本结构,见 otlp/parse.ts,通用);
+ * 语义层(原生 span 名/属性 → canonical GenAI semconv,见 otlp/mappers/<agent>.ts,每 agent 一个薄 mapper)。
+ */
+export interface TraceSpan {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  /** span 起点 / 终点(epoch 毫秒)。 */
+  startMs: number;
+  endMs: number;
+  status?: "ok" | "error" | "unset";
+  /**
+   * 归一后的语义角色(每-agent mapper 据 canonical GenAI semconv 定;view/select 只认它)。
+   * 未经 mapper 或未识别时为 undefined / "other"。
+   */
+  kind?: SpanKind;
+  /** OTLP span 属性(gen_ai.* / tool 名 / token 等),按 key 摊平。raw 属性始终保留供下钻。 */
+  attributes?: Record<string, JsonValue>;
+}
+
+/** 给人 / 给 EVAL.ts 看的 o11y 摘要(注入沙箱 __niceeval__/results.json)。 */
+export interface O11ySummary {
+  totalTurns: number;
+  toolCalls: Record<string, number>;
+  totalToolCalls: number;
+  filesRead: string[];
+  filesModified: string[];
+  shellCommands: { command: string; exitCode?: number; success?: boolean }[];
+  webFetches: { url: string; status?: number; success?: boolean }[];
+  errors: string[];
+  thinkingBlocks: number;
+  compactions: number;
+  durationMs: number;
+  usage: Usage;
+  estimatedCostUSD?: number;
+}