niceeval 0.12.1 → 0.12.2-canary.31

package/src/o11y/otlp/turn-otel.ts

@@ -1,132 +0,0 @@
-// 非沙箱 agent 的 OTLP 接收:run 级共享 receiver + 逐轮 span 归属。
-//
-// 为什么共享:被测应用是长驻进程,只有一条全局 OTel 管线、一个导出目标(OTEL_* env 进程
-// 启动时读一次)——per-attempt receiver 会在第一个 attempt 结束时关掉端口,后续 attempt
-// 的 span 全部丢失。所以粒度跟【被测进程】走:每个 agent(= 一个应用)整个 run 共享一个
-// receiver;沙箱 agent 不走这里(每沙箱独立进程,per-attempt receiver 天然正确)。
-//
-// 共享流之下的逐轮归属(设计:docs/observability.md「span 怎么归属到轮」):
-//   1. traceparent(并发安全):每轮生成新 trace context 经 ctx.telemetry.headers 交给
-//      adapter 随请求带上;应用埋点支持 W3C 传播的,本轮 span 挂在我们给的 traceId 下。
-//   2. 窗口(兜底,仅串行可靠):没见过 traceparent 生效前,同 agent 的轮次串行执行
-//      (并发守卫),把「本轮结束时新到且未被消费的 span」归给本轮。
-//      一旦按 traceId 命中过一次,守卫解除,后续并发随便开、只按 traceId 归属。
-
-import { randomBytes } from "node:crypto";
-import type { TraceSpan } from "../../types.ts";
-import type { TraceReceiver } from "./receiver.ts";
-import { makeTraceReceiver } from "./receiver.ts";
-
-export interface TurnSpans {
-  /** traceparent 命中时是本轮生成值；窗口归属时是实际 span 的 traceId，供 timing 直接挂树。 */
-  traceId: string;
-  /** 归属到本轮的 span。 */
-  spans: TraceSpan[];
-  /** 本轮归属用的是 traceparent 还是时间窗口(日志/守卫用)。 */
-  attribution: "traceparent" | "window";
-}
-
-/** 一个 agent(= 一个被测进程)整个 run 的 OTLP 通道:receiver + 归属状态。 */
-export class AgentOtelChannel {
-  /** traceparent 已确认生效(收到过带我们 traceId 的 span)→ 并发守卫解除。 */
-  private confirmed = false;
-  /** 串行守卫:未确认 traceparent 时,同 agent 的轮次挨个跑(promise 链当互斥锁)。 */
-  private chain: Promise<void> = Promise.resolve();
-  /** 已归属过的 span(spanId),共享流里不能重复分给两轮。 */
-  private readonly consumed = new Set<string>();
-
-  constructor(readonly receiver: TraceReceiver) {}
-
-  /**
-   * 跑一轮:fn 拿到本轮的 W3C trace context headers(spread 进请求头 / 注入子进程 env),
-   * 返回后 settle 接收器、归属本轮 span。未确认 traceparent 前整段(含 fn)串行。
-   */
-  async runTurn<T>(fn: (headers: Readonly<globalThis.Record<string, string>>) => Promise<T>): Promise<{ result: T } & TurnSpans> {
-    const traceId = randomBytes(16).toString("hex");
-    const headers = { traceparent: `00-${traceId}-${randomBytes(8).toString("hex")}-01` };
-
-    const exec = async (): Promise<{ result: T } & TurnSpans> => {
-      const result = await fn(headers);
-      // 给本轮最后一批导出留落地时间(SimpleSpanProcessor 即发;Batch 的 5s 定时兜不住,
-      // 迟到的由 attempt 末尾 sweep 按 traceId 捞回)。
-      await this.receiver.settle(200, 1200);
-      const fresh = this.receiver.collect().filter((s) => !this.consumed.has(s.spanId));
-      const matched = fresh.filter((s) => s.traceId === traceId);
-      let spans: TraceSpan[];
-      let attribution: TurnSpans["attribution"];
-      if (matched.length > 0) {
-        this.confirmed = true;
-        spans = matched;
-        attribution = "traceparent";
-      } else if (this.confirmed) {
-        // 已确认过 traceparent 的通道只按 traceId 归属 —— 窗口在并发下必混流,不回退。
-        spans = [];
-        attribution = "traceparent";
-      } else {
-        spans = fresh;
-        attribution = "window";
-      }
-      for (const s of spans) this.consumed.add(s.spanId);
-      // 窗口归属证明的是“这批新 span 属于本轮”，并不证明应用采用了我们生成的
-      // traceparent。把合成 id 写进 turn 会导致 timing 永远匹配不到实际 span；单 trace
-      // 窗口直接记录真实 id，多 trace 的其余部分仍由 eval.run leftovers 如实保留。
-      const attributedTraceId = attribution === "window" && spans[0] !== undefined ? spans[0].traceId : traceId;
-      return { result, traceId: attributedTraceId, spans, attribution };
-    };
-
-    if (this.confirmed) return exec();
-
-    // 未确认:挂到串行链上。链上的前序轮次结束(含它的 settle)后才开跑,窗口不重叠。
-    const prev = this.chain;
-    const p = prev.then(exec, exec);
-    this.chain = p.then(
-      () => undefined,
-      () => undefined,
-    );
-    return p;
-  }
-
-  /** 是否仍在串行守卫下(日志提示用)。 */
-  get serialized(): boolean {
-    return !this.confirmed;
-  }
-
-  /** attempt 末尾:按本 attempt 的 traceId 集合捞迟到的 span(Batch 导出等)。 */
-  async sweep(traceIds: ReadonlySet<string>): Promise<TraceSpan[]> {
-    if (traceIds.size === 0) return [];
-    await this.receiver.settle(200, 1000);
-    const late = this.receiver
-      .collect()
-      .filter((s) => !this.consumed.has(s.spanId) && traceIds.has(s.traceId));
-    for (const s of late) this.consumed.add(s.spanId);
-    return late;
-  }
-}
-
-/**
- * run 级共享池:每个 agent 一个通道(懒建)。固定端口模式(defineConfig({ telemetry: { port } }))
- * 下全部 agent 共用一个通道 —— 端口只有一个,归属守卫也必须全局共享。
- */
-export class OtelReceiverPool {
-  private readonly channels = new Map<string, Promise<AgentOtelChannel>>();
-
-  constructor(private readonly fixedPort?: number) {}
-
-  channel(agentName: string): Promise<AgentOtelChannel> {
-    const key = this.fixedPort !== undefined ? "*fixed-port*" : agentName;
-    let ch = this.channels.get(key);
-    if (!ch) {
-      ch = makeTraceReceiver(this.fixedPort ?? 0).then((r) => new AgentOtelChannel(r));
-      this.channels.set(key, ch);
-    }
-    return ch;
-  }
-
-  async close(): Promise<void> {
-    const all = [...this.channels.values()];
-    this.channels.clear();
-    await Promise.all(
-      all.map((p) => p.then((ch) => ch.receiver.close()).catch(() => {})),
-    );
-  }
-}

package/src/o11y/parsers/bub.test.ts

@@ -1,30 +0,0 @@
-// cases: docs/engineering/testing/unit/record.md
-// 「Usage、facts 与失败命令证据落盘」桶恒互斥归一:bub tape 的 usage 是 Chat Completions
-// 形状,prompt_tokens_details.cached_tokens 是 prompt_tokens 子集,聚合前扣减;
-// 同对象的 cost 是实测计费,照常累进 costUSD。
-// bug: memory/estimatecost-openai-inclusive-cache-double-billed.md
-
-import { describe, expect, it } from "vitest";
-
-import { parseBubTranscript } from "./bub.ts";
-
-describe("parseBubTranscript usage 归一(OpenAI 口径)", () => {
-  it("cached_tokens 从 prompt_tokens 里扣出,cost 累进实测 costUSD", () => {
-    const line = JSON.stringify({
-      kind: "event",
-      payload: {
-        name: "run",
-        data: {
-          usage: {
-            prompt_tokens: 1000,
-            completion_tokens: 20,
-            prompt_tokens_details: { cached_tokens: 900 },
-            cost: 0.05,
-          },
-        },
-      },
-    });
-    const parsed = parseBubTranscript(line);
-    expect(parsed.usage).toMatchObject({ inputTokens: 100, cacheReadTokens: 900, outputTokens: 20, costUSD: 0.05 });
-  });
-});

package/src/o11y/parsers/bub.ts

@@ -1,264 +0,0 @@
-// Bub(republic tape)transcript 解析器,同时充当「通用 JSONL」兜底解析器。
-//
-// 真实 tape 格式(~/.bub/tapes/<md5(ws)__md5(sess)>.jsonl,按 id 升序逐行):
-//   - kind=message      payload={role, content}                 → 对话
-//   - kind=tool_call    payload={calls:[{id,function:{name,arguments}}]}
-//   - kind=tool_result  payload={results:[...]}                  ← 与上一条 tool_call.calls 按位对齐
-//   - kind=event        payload={name, data}                     name=="run" → data.usage 是该次模型调用用量
-//   - kind=anchor       payload={name, state:{summary?}}         name!="session/start" = tape.handoff(压缩检查点)
-//   - kind=system/error
-// 用量路径(已对真实 tape 校验):payload.data.usage.{prompt_tokens,completion_tokens,total_tokens,
-//   prompt_tokens_details.cached_tokens, cost}。compaction = 非 bootstrap 的 anchor(handoff)条数。
-// 也保留 legacy {type:...} 与无 type/kind 的通用兜底。
-
-import type { StreamEvent, Usage, ToolName, JsonValue } from "../../types.ts";
-import type { ParsedTranscript } from "./index.ts";
-import { GENERIC_VERB_ALIASES, normalizeToolName as normalizeShared } from "../tool-names.ts";
-import { normalizeJsonValue } from "../../shared/json-value.ts";
-
-/** Bub 特有别名(fs.* / web.* 命名空间与裸 read/write/edit)+ 裸动词;通用别名走基表。 */
-export const BUB_TOOL_ALIASES: globalThis.Record<string, ToolName> = {
-  ...GENERIC_VERB_ALIASES,
-  "fs.read": "file_read", fs_read: "file_read", read: "file_read",
-  "fs.write": "file_write", fs_write: "file_write", write: "file_write",
-  "fs.edit": "file_edit", fs_edit: "file_edit", edit: "file_edit",
-  "web.fetch": "web_fetch",
-  "web.search": "web_search",
-  update_todos: "agent_task",
-};
-
-function normalizeToolName(name: string): ToolName {
-  return normalizeShared(name, BUB_TOOL_ALIASES);
-}
-
-function get(obj: unknown, key: string): unknown {
-  return obj && typeof obj === "object" ? (obj as globalThis.Record<string, unknown>)[key] : undefined;
-}
-
-function num(obj: unknown, ...keys: string[]): number {
-  for (const k of keys) {
-    const v = get(obj, k);
-    if (typeof v === "number" && Number.isFinite(v)) return v;
-  }
-  return 0;
-}
-
-function coerceArgs(value: unknown): JsonValue {
-  if (typeof value === "string") {
-    try {
-      const parsed: unknown = JSON.parse(value);
-      return normalizeJsonValue(parsed, value);
-    } catch {
-      return value;
-    }
-  }
-  return normalizeJsonValue(value, {});
-}
-
-function extractText(content: unknown): string {
-  if (typeof content === "string") return content;
-  if (Array.isArray(content)) {
-    const parts: string[] = [];
-    for (const b of content) {
-      if (typeof b === "string") parts.push(b);
-      else {
-        const t = get(b, "text") ?? get(b, "content");
-        if (typeof t === "string") parts.push(t);
-      }
-    }
-    return parts.join("\n");
-  }
-  return "";
-}
-
-export function parseBubTranscript(raw: string | undefined): ParsedTranscript {
-  const events: StreamEvent[] = [];
-  let inputTokens = 0;
-  let outputTokens = 0;
-  let cacheReadTokens = 0;
-  let costUSD = 0;
-  let requests = 0;
-  let compactions = 0;
-  let parseSuccess = true;
-
-  if (!raw || !raw.trim()) {
-    return { events, usage: {}, compactions: 0, parseSuccess: true };
-  }
-
-  // tape 的 tool_result 与上一条 tool_call 按位对齐;记下上一批 callId。
-  let lastCallIds: string[] = [];
-  // 无显式 id 的兜底配对。
-  const pendingCallIds: string[] = [];
-  let synth = 0;
-  const nextSynthId = (): string => `bub_${++synth}`;
-
-  // 从 event/run 的 data.usage(或 legacy 顶层 usage)累加用量。
-  const addUsage = (usage: unknown): void => {
-    if (!usage || typeof usage !== "object") return;
-    const rawInput = num(usage, "input_tokens", "prompt_tokens", "inputTokens");
-    const output = num(usage, "output_tokens", "completion_tokens", "outputTokens");
-    let cache = num(usage, "cached_input_tokens", "cache_read_input_tokens", "cacheReadTokens");
-    if (cache === 0) cache = num(get(usage, "prompt_tokens_details"), "cached_tokens");
-    const cost = num(usage, "cost");
-    if (rawInput === 0 && output === 0 && cache === 0 && cost === 0) return;
-    // Chat Completions 口径的 prompt_tokens 含缓存命中,cached_tokens 是其子集;
-    // 落互斥桶前扣掉(docs/feature/adapters/sdk/bub/cost.md)
-    const input = Math.max(0, rawInput - cache);
-    inputTokens += input;
-    outputTokens += output;
-    cacheReadTokens += cache;
-    costUSD += cost;
-    requests += 1;
-  };
-
-  const emitCall = (originalName: string, input: JsonValue, explicitId: unknown): string => {
-    const callId = explicitId != null ? String(explicitId) : nextSynthId();
-    if (explicitId == null) pendingCallIds.push(callId);
-    events.push({
-      type: "operation.started",
-      operationId: callId,
-      operation: { kind: "tool", name: originalName, input, tool: normalizeToolName(originalName) },
-    });
-    return callId;
-  };
-
-  const emitResult = (output: unknown, success: boolean, explicitId: unknown): void => {
-    const callId = explicitId != null ? String(explicitId) : pendingCallIds.shift() ?? nextSynthId();
-    events.push({
-      type: "operation.finished",
-      operationId: callId,
-      kind: "tool",
-      output: (output ?? null) as JsonValue,
-      status: success ? "completed" : "failed",
-    });
-  };
-
-  for (const line of raw.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-
-    let entry: unknown;
-    try {
-      entry = JSON.parse(trimmed);
-    } catch {
-      parseSuccess = false;
-      continue;
-    }
-
-    try {
-      const type = get(entry, "type");
-      const kind = get(entry, "kind");
-
-      // ── legacy: 顶层 type ──
-      if (typeof type === "string") {
-        addUsage(get(entry, "usage"));
-        if (type === "message") {
-          const role = get(entry, "role") === "user" ? "user" : "assistant";
-          const text = extractText(get(entry, "content") ?? get(entry, "text"));
-          if (text) events.push({ type: "message", role, text });
-        } else if (type === "tool_call") {
-          const name = String(get(entry, "tool_name") ?? get(entry, "name") ?? get(entry, "tool") ?? "unknown");
-          emitCall(name, coerceArgs(get(entry, "args") ?? get(entry, "input")), get(entry, "call_id") ?? get(entry, "id"));
-        } else if (type === "tool_result") {
-          emitResult(get(entry, "result") ?? get(entry, "output") ?? get(entry, "content"), get(entry, "success") !== false && !get(entry, "error"), get(entry, "call_id") ?? get(entry, "tool_call_id") ?? get(entry, "id"));
-        } else if (type === "thinking" || type === "reasoning") {
-          const text = extractText(get(entry, "content") ?? get(entry, "text"));
-          if (text) events.push({ type: "thinking", text });
-        } else if (type === "error") {
-          events.push({ type: "error", message: String(get(entry, "message") ?? get(entry, "error") ?? "error") });
-        }
-        continue;
-      }
-
-      // ── republic tape: kind + payload ──
-      if (typeof kind === "string") {
-        const payload = get(entry, "payload");
-        if (kind === "message") {
-          const role = get(payload, "role") === "user" ? "user" : "assistant";
-          const text = extractText(get(payload, "content") ?? get(payload, "text"));
-          if (text) events.push({ type: "message", role, text });
-        } else if (kind === "tool_call") {
-          // 模型「先说话再调工具」:伴随文本记在 tool_call 的 payload.content 上
-          // (bub 不给它单独的 message 条目)。先还原成 assistant message,再吐 calls,
-          // 否则 transcript 里每次 send 后面都看不到 AI 回复。
-          const accompanying = extractText(get(payload, "content"));
-          if (accompanying) events.push({ type: "message", role: "assistant", text: accompanying });
-          const calls = get(payload, "calls");
-          const list = Array.isArray(calls) ? calls : [payload];
-          lastCallIds = [];
-          for (const call of list) {
-            const fn = get(call, "function") ?? call;
-            const name = String(get(fn, "name") ?? get(call, "name") ?? "unknown");
-            const id = emitCall(name, coerceArgs(get(fn, "arguments") ?? get(call, "args") ?? get(call, "input")), get(call, "id") ?? get(call, "call_id"));
-            lastCallIds.push(id);
-          }
-        } else if (kind === "tool_result") {
-          const results = get(payload, "results");
-          const list = Array.isArray(results) ? results : [get(payload, "result") ?? payload];
-          // republic:results 与上一条 tool_call.calls 按位对齐。
-          list.forEach((r, i) => {
-            const isError = typeof r === "object" && r !== null && "error" in (r as globalThis.Record<string, unknown>);
-            const explicitId = get(r, "tool_call_id") ?? get(r, "id") ?? lastCallIds[i];
-            emitResult(get(r, "output") ?? get(r, "result") ?? get(r, "content") ?? r, !isError, explicitId);
-          });
-        } else if (kind === "event") {
-          const data = get(payload, "data");
-          // 用量:run 事件首选 data.usage;但新版 bub.tape 不一定每轮都发 run/usage,
-          // 所以任何 event 上挂的 usage 也兜底累加(addUsage 自带去重式全零跳过)。
-          addUsage(get(data, "usage"));
-          // 仅在「真失败」时记 error:有 error 字段,或 status 命中失败词。
-          // loop.step 这类控制事件 status=="continue"/"running" 不是错误 —— 别再当 error 刷屏。
-          const status = get(data, "status");
-          const errVal = get(data, "error");
-          const failed =
-            (errVal != null && errVal !== false && errVal !== "") ||
-            (typeof status === "string" && /^(error|failed|aborted|cancell?ed|timeout)$/i.test(status));
-          if (failed) {
-            events.push({ type: "error", message: typeof errVal === "string" && errVal ? errVal : `${String(get(payload, "name"))}: ${String(status)}` });
-          }
-        } else if (kind === "anchor") {
-          // 非 bootstrap 的 anchor = tape.handoff = 一次压缩检查点(供 t.transcript.compactions() 守卫)。
-          const name = String(get(payload, "name") ?? "");
-          if (name && name !== "session/start") {
-            compactions += 1;
-            events.push({ type: "compaction", reason: name });
-          }
-        } else if (kind === "error") {
-          events.push({ type: "error", message: String(get(payload, "message") ?? get(payload, "kind") ?? "error") });
-        }
-        continue;
-      }
-
-      // ── 通用兜底:既无 type 也无 kind ──
-      addUsage(get(entry, "usage"));
-      const role = get(entry, "role");
-      const name = get(entry, "tool") ?? get(entry, "name");
-      if (role === "assistant" || role === "user") {
-        const text = extractText(get(entry, "content") ?? get(entry, "text"));
-        if (text) events.push({ type: "message", role, text });
-      } else if (typeof name === "string") {
-        if (get(entry, "result") !== undefined || get(entry, "output") !== undefined) {
-          emitResult(get(entry, "result") ?? get(entry, "output"), !get(entry, "error"), get(entry, "call_id") ?? get(entry, "id"));
-        } else {
-          emitCall(name, coerceArgs(get(entry, "args") ?? get(entry, "arguments") ?? get(entry, "input")), get(entry, "call_id") ?? get(entry, "id"));
-        }
-      }
-    } catch {
-      parseSuccess = false;
-    }
-  }
-
-  // requests > 0 意味着至少一次 addUsage 真的读到了非零字段;整份 tape 没有任何 usage 时
-  // input/output 也不该垫成 0(见 docs/feature/record/architecture.md#usage)。
-  const usage: Usage = requests > 0 ? { inputTokens, outputTokens } : {};
-  if (cacheReadTokens > 0) usage.cacheReadTokens = cacheReadTokens;
-  if (requests > 0) usage.requests = requests;
-  if (costUSD > 0) usage.costUSD = costUSD;
-
-  return { events, usage, compactions, parseSuccess };
-}
-
-/** 便捷形态:只要 StreamEvent[]。 */
-export function parseBub(raw: string | undefined): StreamEvent[] {
-  return parseBubTranscript(raw).events;
-}