@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.1

package/dist/evals/reconstruct.js

@@ -0,0 +1,322 @@
+import { sanitizeParams, } from "@capybara-hq/plugin-sdk";
+import { AgentDefinitionSchema, normalizeStoredDefinition, } from "../agents/definition.js";
+import { sameModel, selectionAt } from "../conversations/selection.js";
+import { ApiError } from "../errors.js";
+import { mediaRefs, planMedia, } from "../runtime/context/build.js";
+import { assembleModelRequest } from "../runtime/model-call.js";
+import { CHILD_REMOVED_TOOLS } from "../runtime/subagents/definition.js";
+import { PINNED_SKILLS_LIMITS } from "../skills/catalog.js";
+import { canonicalJson } from "./recorded-tools.js";
+import { toEvents } from "./snapshot.js";
+const DRIFT_ORDER = [
+    "system_hash",
+    "tools_changed",
+    "tool_spec_missing",
+    "definition_missing",
+    "pinned_changed",
+    "subagent_changed",
+    "legacy_event",
+    "params_changed",
+    "model_switched",
+    "media_missing",
+    "media_changed",
+    "tool_unmatched",
+];
+function findCall(events, seq) {
+    const call = events.find((e) => e.seq === seq && e.type === "model_call");
+    if (!call) {
+        throw new ApiError("eval.call_not_found", `No model_call event at seq ${seq} in the snapshot`);
+    }
+    return call;
+}
+/** 参与构建该请求的事件：运行时构建时手里的就是 seq ≤ `toSeq` 的事件（`model_call` 本身在其后写入）。 */
+function inputEvents(events, call) {
+    const upTo = Math.min(call.payload.request.toSeq, call.seq - 1);
+    return events.filter((e) => e.seq <= upTo);
+}
+/** 该请求的 agent 版本：所属运行的 `run_started`；没有（不应出现）时用快照运行的版本。 */
+function recordedVersion(s, events, call) {
+    const started = events.find((e) => e.type === "run_started" && e.runId === call.runId);
+    return started?.payload.agentVersion ?? s.agentVersion;
+}
+const sameParams = (a, b) => {
+    const ka = Object.keys(a);
+    return ka.length === Object.keys(b).length && ka.every((k) => Object.is(a[k], b[k]));
+};
+const sameList = (a, b) => (a ?? []).length === (b ?? []).length &&
+    (a ?? []).every((x, i) => x.key === b?.[i]?.key && x.specHash === b?.[i]?.specHash);
+export function reconstructRequest(s, modelCallSeq, opts) {
+    const all = toEvents(s);
+    const call = findCall(all, modelCallSeq);
+    const rec = call.payload.request;
+    const drift = new Set();
+    if (rec.tools === undefined ||
+        rec.params === undefined ||
+        rec.absorbedUpToSeq === undefined ||
+        rec.maxOutputTokens === undefined) {
+        drift.add("legacy_event");
+    }
+    const origVersion = recordedVersion(s, all, call);
+    const version = opts.agentVersion ?? origVersion;
+    const started = subagentStartedOf(all);
+    const templateMoved = started !== undefined &&
+        opts.subagentTemplate !== undefined &&
+        !sameTemplate(started.template, opts.subagentTemplate);
+    const baseline = opts.definition === undefined &&
+        opts.params === undefined &&
+        version === origVersion &&
+        !templateMoved;
+    const stored = s.definitions[String(version)];
+    let agent;
+    if (started !== undefined) {
+        // a sub-agent's request renders the definition and preamble of its subagent_started (the
+        // build reads them from the events); the candidate is only compared with them
+        agent = normalizeStoredDefinition(s.subagent?.definition ?? started.definition);
+        if (!baseline) {
+            const sameVersion = version === origVersion;
+            const candidate = sameVersion
+                ? (opts.definition ?? agent)
+                : (opts.subagentVersion ??
+                    opts.definition ??
+                    (stored ? normalizeStoredDefinition(stored) : undefined));
+            const original = s.definitions[String(origVersion)];
+            if (templateMoved ||
+                // an instructions override reaches every child, whatever the version (batch 14 m5)
+                (!sameVersion &&
+                    opts.definition !== undefined &&
+                    opts.definition.instructions !== agent.instructions) ||
+                (candidate !== undefined &&
+                    subagentChanged(started, agent, candidate, sameVersion, original && normalizeStoredDefinition(original)))) {
+                drift.add("subagent_changed");
+            }
+        }
+    }
+    else {
+        agent = opts.definition ?? (stored && normalizeStoredDefinition(stored));
+    }
+    if (agent === undefined) {
+        drift.add("definition_missing");
+        agent = AgentDefinitionSchema.parse({
+            name: "missing",
+            instructions: "",
+            model: { provider: call.payload.provider, model: call.payload.model },
+        });
+    }
+    // pinned_changed: the candidate pins other skills than the recorded agent version did. Compared
+    // with the original version's definition, not the effective snapshot (which omits stale or
+    // oversized pins), both deduplicated and capped like the run; the snapshot's ids only when that
+    // definition is missing. The stub of definition_missing has no pins, so it is skipped then.
+    if (!baseline && started === undefined && !drift.has("definition_missing")) {
+        const norm = (ids) => [...new Set(ids ?? [])].slice(0, PINNED_SKILLS_LIMITS.count);
+        const origDef = s.definitions[String(origVersion)];
+        let recorded;
+        if (origDef) {
+            recorded = norm(normalizeStoredDefinition(origDef).skills?.pinned);
+        }
+        else {
+            const snap = inputEvents(all, call)
+                .filter((e) => e.type === "skills_catalog")
+                .at(-1);
+            recorded = (snap?.payload.pinned ?? []).map((p) => p.id);
+        }
+        const wanted = norm(agent.skills?.pinned);
+        if (recorded.length !== wanted.length || recorded.some((id, i) => wanted[i] !== id)) {
+            drift.add("pinned_changed");
+        }
+    }
+    // 工具：按记录的顺序从快照取定义；取不到的从请求里去掉（基线与候选一样）
+    const tools = [];
+    const seen = new Set();
+    const kept = (list) => (list ?? []).filter((t) => s.toolSpecs[t.specHash] !== undefined);
+    for (const t of [...(rec.tools ?? []), ...(rec.catalog ?? [])]) {
+        if (seen.has(t.key))
+            continue;
+        seen.add(t.key);
+        const spec = s.toolSpecs[t.specHash];
+        if (spec === undefined) {
+            drift.add("tool_spec_missing");
+            continue;
+        }
+        tools.push({ key: t.key, spec, specHash: t.specHash });
+    }
+    // model_switched: the run used a person's switch (in effect at its run_started) and the
+    // candidate's model replaces it
+    if (!baseline && opts.modelRef !== undefined) {
+        const runStart = all.find((e) => e.type === "run_started" && e.runId === call.runId);
+        const sel = runStart ? selectionAt(all, runStart.seq) : null;
+        if (sel?.model && !sameModel(sel.model, opts.modelRef))
+            drift.add("model_switched");
+    }
+    let params = opts.params;
+    if (params === undefined) {
+        const recorded = rec.params ?? {};
+        const clean = sanitizeParams(opts.model.parameters, recorded);
+        params = clean.value;
+        if (clean.dropped.length > 0 || !sameParams(clean.value, recorded))
+            drift.add("params_changed");
+    }
+    const budget = opts.mediaBudgetBytes ?? rec.mediaBudgetBytes;
+    const { request, meta, mediaTokens } = assembleModelRequest({
+        events: inputEvents(all, call),
+        agent,
+        agentVersion: version,
+        model: opts.model,
+        step: call.payload.step,
+        ...(rec.absorbedUpToSeq !== undefined && { absorbedUpToSeq: rec.absorbedUpToSeq }),
+        params,
+        tools,
+        toolSearch: rec.catalog !== undefined,
+        ...(rec.toolChoice !== undefined && { toolChoice: rec.toolChoice }),
+        ...(opts.mediaInfo !== undefined && { mediaInfo: opts.mediaInfo }),
+        ...(opts.media !== undefined && { media: opts.media }),
+        ...(budget !== undefined && { mediaBudgetBytes: budget }),
+        defaultMaxOutputTokens: rec.maxOutputTokens ?? opts.model.maxOutputTokens,
+    });
+    if (baseline) {
+        if (meta.systemHash !== rec.systemHash)
+            drift.add("system_hash");
+        // 与记录比较时排除快照里缺定义的工具（已记 tool_spec_missing）
+        if (rec.tools !== undefined &&
+            (!sameList(meta.tools, kept(rec.tools)) ||
+                !sameList(meta.catalog, rec.catalog === undefined ? undefined : kept(rec.catalog)))) {
+            drift.add("tools_changed");
+        }
+    }
+    // 媒体：按附件 id 与记录比较
+    if (meta.media !== undefined && (rec.media === undefined || rec.mediaBudgetBytes === undefined)) {
+        drift.add("legacy_event");
+    }
+    const recordedMedia = new Map((rec.media ?? []).map((m) => [m.attachmentId, m]));
+    for (const m of meta.media ?? []) {
+        const was = recordedMedia.get(m.attachmentId);
+        if (m.outcome === "unavailable" && was?.outcome !== "unavailable") {
+            drift.add("media_missing");
+        }
+        else if (baseline &&
+            rec.media !== undefined &&
+            (was === undefined || was.outcome !== m.outcome || was.etag !== m.etag)) {
+            drift.add("media_changed");
+        }
+    }
+    if (baseline && rec.media !== undefined) {
+        const now = new Set((meta.media ?? []).map((m) => m.attachmentId));
+        if (rec.media.some((m) => !now.has(m.attachmentId)))
+            drift.add("media_changed");
+    }
+    return { request, meta, drift: DRIFT_ORDER.filter((r) => drift.has(r)), mediaTokens };
+}
+/** The `subagent_started` of a sub-agent's (child) log, or undefined for any other conversation. */
+export function subagentStartedOf(events) {
+    const e = events.find((x) => x.type === "subagent_started");
+    return e?.type === "subagent_started" ? e.payload : undefined;
+}
+const sameTemplate = (logged, t) => logged.source !== "plugin" || (logged.id === t.id && logged.version === t.version);
+const pinsOf = (d) => [...new Set(d.skills?.pinned ?? [])].slice(0, PINNED_SKILLS_LIMITS.count);
+/** Tool refs as a sub-agent gets them without template narrowing (general and org agents). */
+const childToolsOf = (d) => canonicalJson(d.tools.filter((t) => t.kind !== "builtin" || !CHILD_REMOVED_TOOLS.includes(t.name)));
+/** The parent's selection a template child depends on: its tools, org skills and roster entry. */
+const parentSide = (d, started) => canonicalJson({
+    tools: d.tools,
+    orgSkills: d.skills?.orgSkills ?? [],
+    listed: started.template.source === "general"
+        ? d.subagents?.general !== false
+        : (d.subagents?.templates ?? []).includes(started.template.id ?? ""),
+});
+/**
+ * Does a candidate change what a sub-agent (child) case would run (`subagent_changed`, spec
+ * §15.1)? Pure, from the snapshot alone, comparing the model-visible parts of the effective
+ * definition (instructions, pinned skills, tools) with the logged one (`logged`, from its
+ * `subagent_started`):
+ * - the same agent version (`sameVersion`): the candidate is the logged definition with
+ *   overrides (instructions, model, params), compared field by field;
+ * - another version of the case's agent: for an org agent child (`source: "agent"`) that agent's
+ *   version, as dispatch would use it (its own tools minus the child-removed ones); otherwise the
+ *   parent's version: `general` takes its instructions and tools and pins nothing, `inline` its
+ *   template of the same name (gone = changed), and a `plugin` template does not depend on the
+ *   parent version (its own version is {@link ReconstructOptions.subagentTemplate}). Template
+ *   narrowing that needs the database (inline and plugin tool allowlists, skill names) is not
+ *   recomputed: those compare instructions only.
+ * Models and params are not part of it: they are candidate axes of their own.
+ */
+export function subagentChanged(started, logged, candidate, sameVersion, 
+/** The parent's definition at the case's version (the snapshot's), for template children. */
+original) {
+    const differs = (instructions, pinned, tools) => instructions !== logged.instructions ||
+        (pinned !== null && canonicalJson(pinned) !== canonicalJson(pinsOf(logged))) ||
+        (tools !== null && tools !== childToolsOf(logged));
+    if (sameVersion) {
+        return differs(candidate.instructions, pinsOf(candidate), canonicalJson(candidate.tools));
+    }
+    switch (started.template.source) {
+        case "agent":
+            return differs(candidate.instructions, pinsOf(candidate), childToolsOf(candidate));
+        case "general":
+            return (differs(candidate.instructions, [], childToolsOf(candidate)) ||
+                (original !== undefined && parentSide(original, started) !== parentSide(candidate, started)));
+        case "inline": {
+            const t = candidate.subagents?.inline?.find((x) => x.name === started.agentType);
+            if (!t)
+                return true;
+            // conservative: any difference of the template between the two versions counts
+            const o = original?.subagents?.inline?.find((x) => x.name === started.agentType);
+            if (o !== undefined)
+                return canonicalJson(o) !== canonicalJson(t);
+            return differs(t.instructions ?? candidate.instructions, null, null);
+        }
+        case "plugin":
+            return (original !== undefined && parentSide(original, started) !== parentSide(candidate, started));
+    }
+}
+/**
+ * `subagent_changed` for a whole child case (full-run replay): the candidate (`definition` at
+ * `agentVersion`, plus a plugin template's current version) against the case's logged sub-agent.
+ * False for any other case and for an unchanged candidate.
+ */
+export function subagentCaseChanged(s, c) {
+    const started = subagentStartedOf(toEvents(s));
+    if (started === undefined || !c.changed)
+        return false;
+    if (c.subagentTemplate !== undefined && !sameTemplate(started.template, c.subagentTemplate)) {
+        return true;
+    }
+    const logged = normalizeStoredDefinition(s.subagent?.definition ?? started.definition);
+    const sameVersion = c.agentVersion === s.agentVersion;
+    const candidate = sameVersion ? c.definition : (c.subagentVersion ?? c.definition);
+    // an instructions override reaches every child, whatever the version (batch 14 m5): the
+    // rendered definition is the logged one with the candidate's instructions
+    if (!sameVersion && c.definition.instructions !== logged.instructions)
+        return true;
+    const original = s.definitions[String(s.agentVersion)];
+    return subagentChanged(started, logged, candidate, sameVersion, original && normalizeStoredDefinition(original));
+}
+/** 还原该请求要查的附件 id（该模型用得到的引用，去重、按出现顺序；见 `mediaRefs`）。 */
+export function snapshotMediaRefs(s, modelCallSeq, model) {
+    const all = toEvents(s);
+    return mediaRefs(inputEvents(all, findCall(all, modelCallSeq)), model);
+}
+/**
+ * 按快照自己的 `orgId` / `conversationId` 取还原该请求要用的附件（C3：媒体不进快照，回放时取原附件）：
+ * 先查行信息，按 `planMedia`（与运行时同一规则；预算缺省用记录的 `mediaBudgetBytes`）只取要发字节的，
+ * `text/*` 只取开头。结果直接作 {@link ReconstructOptions} 的 `mediaInfo` / `media`。
+ * 原对话或附件已删除 → 行不在结果里，还原时按「不再可用」占位（drift 记 `media_missing`）；
+ * 暂时故障（存储不可用）由来源抛出，调用方按可重试处理。
+ */
+export async function loadSnapshotMedia(s, modelCallSeq, model, source, mediaBudgetBytes) {
+    const all = toEvents(s);
+    const call = findCall(all, modelCallSeq);
+    const events = inputEvents(all, call);
+    const refs = mediaRefs(events, model);
+    const media = new Map();
+    if (refs.length === 0)
+        return { mediaInfo: new Map(), media };
+    const mediaInfo = await source.info(s.orgId, s.conversationId, refs);
+    const budget = mediaBudgetBytes ?? call.payload.request.mediaBudgetBytes;
+    const plan = planMedia(events, model, mediaInfo, budget);
+    if (plan.fetch.length > 0) {
+        const caps = new Map(plan.fetch.flatMap((id) => (plan.caps.has(id) ? [[id, plan.caps.get(id)]] : [])));
+        const got = await source.fetch(s.orgId, s.conversationId, plan.fetch, caps.size > 0 ? caps : undefined);
+        for (const id of plan.fetch)
+            media.set(id, got.get(id) ?? null);
+    }
+    return { mediaInfo, media };
+}

package/dist/evals/recorded-tools.d.ts

@@ -0,0 +1,54 @@
+import type { EventLog } from "../conversations/event-log.js";
+import { type ToolPipeline, type ToolStepContext } from "../tools/pipeline.js";
+import type { CaseSnapshot } from "./snapshot.js";
+/**
+ * 整轮回放的工具管线（C6 / Review Focus 3）：只回放原运行记录的工具结果，**从不执行任何工具**——
+ * 不经过策略、审批、沙箱、连接器，也不调用任何处理函数。每个调用按 (toolKey, 规范化参数) 精确匹配第一条未用的记录；
+ * 匹配不上时写英文错误结果 {@link REPLAY_NO_RECORDING}（`isError: true`），什么都不执行。
+ */
+/** 没有匹配记录时模型看到的工具结果（模型可见，英文）。 */
+export declare const REPLAY_NO_RECORDING = "[replay] No recorded result matches this tool call (same tool and arguments); the tool was not executed during replay.";
+/** 原运行里一次有结果的工具调用。 */
+export interface RecordedCall {
+    toolKey: string;
+    /** 参数的规范化 JSON（对象键递归排序，见 {@link canonicalJson}；不含 {@link matchKey} 忽略的字段）。 */
+    inputKey: string;
+    content: string;
+    isError: boolean;
+    /** 原结果里的 `tool_search` 已加载工具（照抄：之后请求的已加载集合由日志还原）。 */
+    loadedTools?: string[];
+    contentTypes?: string[];
+    /** 原 `tool_call_started` 的定义指纹（没有开始事件的调用为空串）。 */
+    specHash: string;
+}
+/** 规范化 JSON：对象键递归按码点排序，数组保持顺序；`undefined` 按 JSON 规则省略。 */
+export declare function canonicalJson(v: unknown): string;
+/** The matching key of a call: the canonical JSON of its input without the ignored fields. */
+export declare function matchKey(toolKey: string, input: unknown): string;
+/**
+ * 快照运行里有结果的工具调用，按调用出现的顺序（assistant 消息的 seq、块的顺序）。
+ * 没有 `tool_result` 的调用（运行中止等）不算记录。
+ */
+export declare function recordedCalls(s: CaseSnapshot): RecordedCall[];
+export interface RecordedToolPipelineDeps {
+    eventLog: EventLog;
+    recordings: RecordedCall[];
+    /**
+     * 每步处理工具调用之前调用（运行仍在执行、仍持有认领等）：返回 `stop` 时本步尚无结果的调用写
+     * {@link RUN_STOPPED_TEXT}（保持「调用都有结果」），调用方负责让 loop 停下（中止信号）。
+     */
+    beforeStep(step: number): Promise<"continue" | "stop">;
+}
+export declare class RecordedToolPipeline implements Pick<ToolPipeline, "runStep"> {
+    private readonly d;
+    private readonly used;
+    private misses;
+    constructor(d: RecordedToolPipelineDeps);
+    /** 至今没有匹配到记录的调用数。 */
+    unmatched(): number;
+    runStep(ctx: ToolStepContext): Promise<{
+        kind: "done";
+    }>;
+    /** 第一条未用、且工具与参数都相同的记录。 */
+    private take;
+}

package/dist/evals/recorded-tools.js

@@ -0,0 +1,167 @@
+import { assistantIdemKey } from "../runtime/model-call.js";
+import { RUN_STOPPED_TEXT } from "../tools/pipeline.js";
+/**
+ * 整轮回放的工具管线（C6 / Review Focus 3）：只回放原运行记录的工具结果，**从不执行任何工具**——
+ * 不经过策略、审批、沙箱、连接器，也不调用任何处理函数。每个调用按 (toolKey, 规范化参数) 精确匹配第一条未用的记录；
+ * 匹配不上时写英文错误结果 {@link REPLAY_NO_RECORDING}（`isError: true`），什么都不执行。
+ */
+/** 没有匹配记录时模型看到的工具结果（模型可见，英文）。 */
+export const REPLAY_NO_RECORDING = "[replay] No recorded result matches this tool call (same tool and arguments); the tool was not executed during replay.";
+/** 规范化 JSON：对象键递归按码点排序，数组保持顺序；`undefined` 按 JSON 规则省略。 */
+export function canonicalJson(v) {
+    const norm = (x) => {
+        if (Array.isArray(x))
+            return x.map(norm);
+        if (x !== null && typeof x === "object") {
+            const o = x;
+            const out = {};
+            for (const k of Object.keys(o).sort())
+                out[k] = norm(o[k]);
+            return out;
+        }
+        return x;
+    };
+    return JSON.stringify(norm(v) ?? null);
+}
+/**
+ * Input fields left out when matching, per tool: display-only arguments the model writes that do not
+ * change what the tool does. `sandbox_bash`'s `description` became required in UI refresh B, so a
+ * recording without it and a replayed call with one (or a different wording) still match.
+ */
+const MATCH_IGNORED_FIELDS = {
+    sandbox_bash: ["description"],
+};
+/** The matching key of a call: the canonical JSON of its input without the ignored fields. */
+export function matchKey(toolKey, input) {
+    const ignored = MATCH_IGNORED_FIELDS[toolKey];
+    if (ignored === undefined ||
+        input === null ||
+        typeof input !== "object" ||
+        Array.isArray(input)) {
+        return canonicalJson(input);
+    }
+    const rest = { ...input };
+    for (const f of ignored)
+        delete rest[f];
+    return canonicalJson(rest);
+}
+/**
+ * 快照运行里有结果的工具调用，按调用出现的顺序（assistant 消息的 seq、块的顺序）。
+ * 没有 `tool_result` 的调用（运行中止等）不算记录。
+ */
+export function recordedCalls(s) {
+    const own = s.events.filter((e) => e.runId === s.runId);
+    const results = new Map();
+    const started = new Map();
+    for (const e of own) {
+        if (e.type === "tool_result") {
+            const p = e.payload;
+            results.set(p.callId, p);
+        }
+        else if (e.type === "tool_call_started") {
+            const p = e.payload;
+            if (!started.has(p.callId))
+                started.set(p.callId, p.specHash);
+        }
+    }
+    const out = [];
+    for (const e of own) {
+        if (e.type !== "assistant_message")
+            continue;
+        const content = e.payload.content;
+        for (const b of content) {
+            if (b.type !== "tool_call")
+                continue;
+            const call = b;
+            const r = results.get(call.id);
+            if (!r)
+                continue;
+            out.push({
+                toolKey: call.name,
+                inputKey: matchKey(call.name, call.input),
+                content: r.content,
+                isError: r.isError,
+                ...(r.loadedTools !== undefined && { loadedTools: r.loadedTools }),
+                ...(r.contentTypes !== undefined && { contentTypes: r.contentTypes }),
+                specHash: started.get(call.id) ?? "",
+            });
+        }
+    }
+    return out;
+}
+/** 与真实管线相同的幂等键（同一调用重入时不重复写）。 */
+const idem = {
+    started: (runId, callId) => `tool_started:${runId}:${callId}`,
+    result: (runId, callId) => `tool_result:${runId}:${callId}`,
+};
+export class RecordedToolPipeline {
+    d;
+    used = new Set();
+    misses = 0;
+    constructor(d) {
+        this.d = d;
+    }
+    /** 至今没有匹配到记录的调用数。 */
+    unmatched() {
+        return this.misses;
+    }
+    async runStep(ctx) {
+        const { eventLog } = this.d;
+        const msg = await eventLog.findByIdem(ctx.conversationId, assistantIdemKey(ctx.runId, ctx.step));
+        if (msg?.type !== "assistant_message")
+            return { kind: "done" };
+        const calls = msg.payload.content.flatMap((b) => b.type === "tool_call" ? [{ id: b.id, name: b.name, input: b.input }] : []);
+        if (calls.length === 0)
+            return { kind: "done" };
+        const stop = (await this.d.beforeStep(ctx.step)) === "stop";
+        for (const call of calls) {
+            if (await eventLog.findByIdem(ctx.conversationId, idem.result(ctx.runId, call.id)))
+                continue;
+            const result = (p) => ({
+                type: "tool_result",
+                runId: ctx.runId,
+                idemKey: idem.result(ctx.runId, call.id),
+                payload: { step: ctx.step, callId: call.id, toolKey: call.name, ...p },
+            });
+            if (stop) {
+                await eventLog.append(ctx.conversationId, [
+                    result({ content: RUN_STOPPED_TEXT, isError: true }),
+                ]);
+                continue;
+            }
+            const rec = this.take(call.name, matchKey(call.name, call.input));
+            if (!rec)
+                this.misses++;
+            await eventLog.append(ctx.conversationId, [
+                {
+                    type: "tool_call_started",
+                    runId: ctx.runId,
+                    idemKey: idem.started(ctx.runId, call.id),
+                    payload: {
+                        step: ctx.step,
+                        callId: call.id,
+                        toolKey: call.name,
+                        specHash: rec?.specHash ?? "",
+                    },
+                },
+                result(rec
+                    ? {
+                        content: rec.content,
+                        isError: rec.isError,
+                        ...(rec.loadedTools !== undefined && { loadedTools: rec.loadedTools }),
+                        ...(rec.contentTypes !== undefined && { contentTypes: rec.contentTypes }),
+                    }
+                    : { content: REPLAY_NO_RECORDING, isError: true }),
+            ]);
+        }
+        return { kind: "done" };
+    }
+    /** 第一条未用、且工具与参数都相同的记录。 */
+    take(toolKey, inputKey) {
+        const i = this.d.recordings.findIndex((r, j) => !this.used.has(j) && r.toolKey === toolKey && r.inputKey === inputKey);
+        if (i < 0)
+            return undefined;
+        this.used.add(i);
+        return this.d.recordings[i];
+    }
+}

package/dist/evals/run-replay.d.ts

@@ -0,0 +1,86 @@
+import { type ContentBlock, type Logger, type ModelUsage, type StopReason } from "@capybara-hq/plugin-sdk";
+import type { KernelConfig } from "../config/schema.js";
+import type { EventLog } from "../conversations/event-log.js";
+import type { KernelDb } from "../db/kernel-schema.js";
+import type { KyselyCheckpointSaver } from "../runtime/checkpointer.js";
+import type { LoopDeps } from "../runtime/langgraph-loop.js";
+import type { AgentLoop, LoopOutcome } from "../runtime/loop.js";
+import { type MediaSourceDeps } from "../runtime/model-call.js";
+import type { ProviderRegistry } from "../runtime/providers.js";
+import type { CaseSnapshot } from "./snapshot.js";
+import type { ResolvedCandidate } from "./step-replay.js";
+/**
+ * 整轮回放（D2 / C6）：在 `kind = 'eval'` 的隐藏对话里跑**同一个** `LangGraphLoop`，工具管线换成
+ * {@link RecordedToolPipeline}（只回放记录的结果，不执行任何工具、沙箱、连接器或审批）。
+ *
+ * - 种子：原运行 `run_started` 之前的事件按原 seq 插入（`last_seq` 停在其前一位，新的 `run_started` 因此与原来同 seq），
+ *   原运行开始时的 `skills_catalog` 照抄；然后写 `runs` 行与 `run_started`，`agent` 用候选定义。
+ * - 插话：原运行中途吸收的 user_message，在「原吸收步骤 ≤ 当前步骤」时追加进评测对话并交给 loop。
+ * - 计费（C7，与单步回放同一套）：每次模型请求（含压缩、重试、兜底）发出前查运行仍在执行、仍持有认领；
+ *   收完按真实用量、没收完按估算 `consumeTokens`（越限即 `capped`，下一次请求前停下）。
+ * - 附件按快照自己的组织与原对话取（C3）。
+ * 崩溃后由调用方删掉旧的评测对话（{@link deleteEvalConversation}），从头重跑。
+ */
+export interface RunReplayResult {
+    conversationId: string;
+    outcome: LoopOutcome["kind"];
+    toolCalls: {
+        name: string;
+        input: unknown;
+    }[];
+    finalContent: ContentBlock[];
+    stopReason: StopReason | null;
+    usage: ModelUsage;
+    steps: number;
+    unmatched: number;
+}
+/** 原运行（基线）的同形摘要。 */
+export type RunBaseline = Omit<RunReplayResult, "conversationId" | "outcome" | "unmatched">;
+/** 测试钩子。 */
+export interface EvalReplayHooks {
+    /** 种子已写入（评测对话已记到结果上）、loop 开始之前调用：抛错即模拟此时崩溃。 */
+    afterRunSeeded?(evalRunId: string, conversationId: string): Promise<void>;
+    /** 每次请求前的检查（查运行状态与认领）之前调用：抛错即模拟检查时的 DB 故障。 */
+    beforeCheck?(evalRunId: string): Promise<void>;
+    /** 每次计费（`consumeTokens`）之前调用：抛错即模拟计费时的 DB 故障。 */
+    beforeCharge?(evalRunId: string): Promise<void>;
+    /** 删除评测对话之前调用：抛错即模拟删除时的 DB 故障。 */
+    beforeDeleteConversation?(conversationId: string): Promise<void>;
+    /** 每次评委请求之前调用（worker）：抛错即模拟「结果已 done、评委未做」时崩溃。 */
+    beforeJudge?(evalRunId: string): Promise<void>;
+}
+export interface RunReplayDeps {
+    db: KernelDb;
+    eventLog: EventLog;
+    config: () => KernelConfig;
+    log: Logger;
+    registry: Pick<ProviderRegistry, "resolve">;
+    media?: MediaSourceDeps;
+    checkpoints?: Pick<KyselyCheckpointSaver, "deleteThread">;
+    /**
+     * 构建评测用的 loop：与运行时同一个 `LangGraphLoop`（同一工厂），只换掉工具管线、模型注册表（计费）与附件来源。
+     */
+    loop(d: Pick<LoopDeps, "toolPipeline" | "registry" | "media" | "skillForks">): AgentLoop;
+    /** 本次执行是否仍持有结果的认领（失去认领 → 不再发请求）；不给 = 不检查。 */
+    stillClaimed?: () => Promise<boolean>;
+    /**
+     * 种子事务内调用：把评测对话记到结果上（以认领为条件）。返回 false = 已失去认领，种子回滚、抛 {@link ReplayClaimLost}。
+     */
+    onSeeded?: (trx: KernelDb, conversationId: string) => Promise<boolean>;
+    hooks?: EvalReplayHooks;
+}
+/** 写种子时发现已失去认领或运行已不在执行：本次执行停下，不留评测对话。 */
+export declare class ReplayClaimLost extends Error {
+    constructor();
+}
+/** 整轮回放被停下的原因（`run_aborted.detail`）。 */
+export type StopDetail = "cancelled" | "capped" | "too_many_errors" | "claim_lost" | "shutdown";
+/** 原运行（快照）的摘要，作为整轮回放的基线。 */
+export declare function runBaseline(s: CaseSnapshot): RunBaseline;
+/**
+ * 删掉一个评测对话（崩溃重跑前清掉上一次的残留）：先删它各次运行的 checkpoint，再删对话（事件、运行随之级联删除）。
+ * 只删 `kind = 'eval'` 的对话。
+ */
+export declare function deleteEvalConversation(d: Pick<RunReplayDeps, "db" | "checkpoints" | "hooks">, conversationId: string): Promise<void>;
+/** 整轮回放一个用例。见文件头注释。 */
+export declare function replayRun(d: RunReplayDeps, s: CaseSnapshot, candidate: ResolvedCandidate, evalRunId: string, signal: AbortSignal): Promise<RunReplayResult>;