@capybara-hq/kernel 0.1.0-rc.0 → 0.1.0-rc.10

package/dist/evals/recorded-tools.js

@@ -1,4 +1,7 @@
+import { environmentPathsChange } from "../runtime/context/environment-paths.js";
 import { assistantIdemKey } from "../runtime/model-call.js";
+import { MATCH_IGNORED_FIELDS } from "../tools/builtin-names.js";
+import { isLegacyPrefixedKey, legacyToolInput, legacyToolKey, } from "../tools/legacy-keys.js";
 import { RUN_STOPPED_TEXT } from "../tools/pipeline.js";
 /**
  * 整轮回放的工具管线（C6 / Review Focus 3）：只回放原运行记录的工具结果，**从不执行任何工具**——
@@ -7,6 +10,85 @@ import { RUN_STOPPED_TEXT } from "../tools/pipeline.js";
  */
 /** 没有匹配记录时模型看到的工具结果（模型可见，英文）。 */
 export const REPLAY_NO_RECORDING = "[replay] No recorded result matches this tool call (same tool and arguments); the tool was not executed during replay.";
+/**
+ * The connectors and extensions an rc `<x>__<tool>` key can name, from positive evidence in the
+ * snapshot only (a head with no evidence, or evidence of both kinds, stays unmapped, so its
+ * recorded calls are unmatched, which is a safe miss):
+ * - a connector: the tool definition the key was recorded with (`toolSpecs`, found by the specHash of
+ *   its `tool_call_started` or its `model_call` tools / catalog entry) has `connectorData`;
+ * - an extension: that definition exists without `connectorData` and its name is the key's tool
+ *   part, or an rc-era agent definition names exactly that tool as a `{ kind: "plugin", plugin: x, name }`
+ *   ref.
+ */
+export function legacyScopeOf(s) {
+    const hashes = new Map();
+    const note = (key, hash) => {
+        if (!isLegacyPrefixedKey(key))
+            return;
+        const set = hashes.get(key) ?? new Set();
+        if (hash !== undefined && hash !== "")
+            set.add(hash);
+        hashes.set(key, set);
+    };
+    for (const e of s.events) {
+        if (e.type === "tool_call_started") {
+            const p = e.payload;
+            note(p.toolKey, p.specHash);
+        }
+        else if (e.type === "model_call") {
+            const r = e.payload.request;
+            for (const t of [...(r?.tools ?? []), ...(r?.catalog ?? [])])
+                note(t.key, t.specHash);
+        }
+    }
+    const extensionNames = new Set();
+    const connectorSlugs = new Set();
+    const planted = new Set();
+    // rc-era snapshots hold definitions from before tools were on by default: an enabled `tools`
+    // list of refs (tools default-on spec §2)
+    for (const def of Object.values(s.definitions)) {
+        const refs = def.tools;
+        if (!Array.isArray(refs))
+            continue;
+        for (const t of refs) {
+            if (t?.kind === "plugin" && typeof t.plugin === "string" && typeof t.name === "string") {
+                planted.add(`${t.plugin}__${t.name}`);
+            }
+        }
+    }
+    for (const [key, set] of hashes) {
+        const i = key.indexOf("__");
+        const head = key.slice(0, i);
+        const tool = key.slice(i + 2);
+        for (const h of set) {
+            const spec = s.toolSpecs[h];
+            if (spec === undefined)
+                continue;
+            if (spec.connectorData !== undefined) {
+                connectorSlugs.add(head);
+            }
+            else if (spec.name === tool) {
+                extensionNames.add(head);
+            }
+        }
+    }
+    for (const key of planted) {
+        if (hashes.has(key))
+            extensionNames.add(key.slice(0, key.indexOf("__")));
+    }
+    return { connectorSlugs, extensionNames };
+}
+/**
+ * A recorded call in the current names: an rc builtin or `<x>__<tool>` key becomes its current
+ * key with the input's parameters renamed. Only the recorded side is ever normalized; a call
+ * already in current names (or one that cannot be mapped) is returned as is.
+ */
+export function currentCall(name, input, scope) {
+    const key = legacyToolKey(name, scope);
+    if (key === null)
+        return { name, input, legacy: false };
+    return { name: key, input: legacyToolInput(name, input), legacy: true };
+}
 /** 规范化 JSON：对象键递归按码点排序，数组保持顺序；`undefined` 按 JSON 规则省略。 */
 export function canonicalJson(v) {
     const norm = (x) => {
@@ -23,14 +105,6 @@ export function canonicalJson(v) {
     };
     return JSON.stringify(norm(v) ?? null);
 }
-/**
- * Input fields left out when matching, per tool: display-only arguments the model writes that do not
- * change what the tool does. `sandbox_bash`'s `description` became required in UI refresh B, so a
- * recording without it and a replayed call with one (or a different wording) still match.
- */
-const MATCH_IGNORED_FIELDS = {
-    sandbox_bash: ["description"],
-};
 /** The matching key of a call: the canonical JSON of its input without the ignored fields. */
 export function matchKey(toolKey, input) {
     const ignored = MATCH_IGNORED_FIELDS[toolKey];
@@ -64,10 +138,29 @@ export function recordedCalls(s) {
                 started.set(p.callId, p.specHash);
         }
     }
+    // an environment_paths event belongs to the step of the assistant message before it, and holds
+    // for that step's calls and every later step's (until the next one)
+    const pathsOf = new Map();
+    const assistants = own.filter((e) => e.type === "assistant_message").map((e) => e.seq);
+    let current;
+    let next = 0;
+    for (const [i, a] of assistants.entries()) {
+        const end = assistants[i + 1] ?? Number.POSITIVE_INFINITY;
+        for (; next < own.length && own[next].seq < end; next++) {
+            const e = own[next];
+            if (e.type === "environment_paths" && e.seq > a) {
+                current = e.payload.roots;
+            }
+        }
+        if (current !== undefined)
+            pathsOf.set(a, current);
+    }
+    const scope = legacyScopeOf(s);
     const out = [];
     for (const e of own) {
         if (e.type !== "assistant_message")
             continue;
+        const paths = pathsOf.get(e.seq);
         const content = e.payload.content;
         for (const b of content) {
             if (b.type !== "tool_call")
@@ -76,14 +169,17 @@ export function recordedCalls(s) {
             const r = results.get(call.id);
             if (!r)
                 continue;
+            const cur = currentCall(call.name, call.input, scope);
             out.push({
-                toolKey: call.name,
-                inputKey: matchKey(call.name, call.input),
+                toolKey: cur.name,
+                inputKey: matchKey(cur.name, cur.input),
+                ...(cur.legacy && { legacy: true }),
                 content: r.content,
                 isError: r.isError,
                 ...(r.loadedTools !== undefined && { loadedTools: r.loadedTools }),
                 ...(r.contentTypes !== undefined && { contentTypes: r.contentTypes }),
                 specHash: started.get(call.id) ?? "",
+                ...(paths !== undefined && { environmentPaths: paths }),
             });
         }
     }
@@ -98,9 +194,14 @@ export class RecordedToolPipeline {
     d;
     used = new Set();
     misses = 0;
+    legacyHits = 0;
     constructor(d) {
         this.d = d;
     }
+    /** Calls so far matched with a recording made under release-candidate tool names. */
+    legacyMatches() {
+        return this.legacyHits;
+    }
     /** 至今没有匹配到记录的调用数。 */
     unmatched() {
         return this.misses;
@@ -132,7 +233,14 @@ export class RecordedToolPipeline {
             const rec = this.take(call.name, matchKey(call.name, call.input));
             if (!rec)
                 this.misses++;
+            // the original run had told the model the instance's directories by this call: so does the
+            // replay, when the eval conversation's latest record differs (as the sandbox tools write it)
+            const paths = rec?.environmentPaths;
+            const change = paths === undefined
+                ? null
+                : environmentPathsChange(await eventLog.latestOfType(ctx.conversationId, "environment_paths"), paths, ctx.runId);
             await eventLog.append(ctx.conversationId, [
+                ...(change ? [change] : []),
                 {
                     type: "tool_call_started",
                     runId: ctx.runId,
@@ -162,6 +270,8 @@ export class RecordedToolPipeline {
         if (i < 0)
             return undefined;
         this.used.add(i);
+        if (this.d.recordings[i]?.legacy === true)
+            this.legacyHits++;
         return this.d.recordings[i];
     }
 }

package/dist/evals/run-replay.d.ts

@@ -1,21 +1,30 @@
 import { type ContentBlock, type Logger, type ModelUsage, type StopReason } from "@capybara-hq/plugin-sdk";
-import type { KernelConfig } from "../config/schema.js";
-import type { EventLog } from "../conversations/event-log.js";
-import type { KernelDb } from "../db/kernel-schema.js";
+import { type ModelRef } from "../agents/definition.js";
+import { EventLog } from "../conversations/event-log.js";
+import type { KernelSchemaDb } from "../db/kernel-schema.js";
+import type { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
 import type { KyselyCheckpointSaver } from "../runtime/checkpointer.js";
 import type { LoopDeps } from "../runtime/langgraph-loop.js";
 import type { AgentLoop, LoopOutcome } from "../runtime/loop.js";
 import { type MediaSourceDeps } from "../runtime/model-call.js";
 import type { ProviderRegistry } from "../runtime/providers.js";
-import type { CaseSnapshot } from "./snapshot.js";
+import type { RunFinish } from "../runtime/run-finish.js";
+import type { CaseSnapshot, SnapshotEvent } from "./snapshot.js";
 import type { ResolvedCandidate } from "./step-replay.js";
 /**
  * 整轮回放（D2 / C6）：在 `kind = 'eval'` 的隐藏对话里跑**同一个** `LangGraphLoop`，工具管线换成
  * {@link RecordedToolPipeline}（只回放记录的结果，不执行任何工具、沙箱、连接器或审批）。
  *
  * - 种子：原运行 `run_started` 之前的事件按原 seq 插入（`last_seq` 停在其前一位，新的 `run_started` 因此与原来同 seq），
- *   原运行开始时的 `skills_catalog` 照抄；然后写 `runs` 行与 `run_started`，`agent` 用候选定义。
- * - 插话：原运行中途吸收的 user_message，在「原吸收步骤 ≤ 当前步骤」时追加进评测对话并交给 loop。
+ *   原运行开始时的 `skills_catalog` 与 `run_context` 照抄；然后写 `runs` 行与 `run_started`，`agent` 用候选定义。
+ * - 插话：原运行中途吸收的可吸收事件（user_message, and finished background tasks: background Bash
+ *   spec §4.4），在「原吸收步骤 ≤ 当前步骤」时追加进评测对话并交给 loop（a recorded finish is
+ *   appended as recorded: no task starts, polls or stops; likewise a sub-agent's delivery, agent map
+ *   spec §8: no child is dispatched, sent to or woken）。
+ * - A wake run (its boundary a finished background task or a sub-agent's delivery, not a message)
+ *   is seeded from that event.
  * - 计费（C7，与单步回放同一套）：每次模型请求（含压缩、重试、兜底）发出前查运行仍在执行、仍持有认领；
  *   收完按真实用量、没收完按估算 `consumeTokens`（越限即 `capped`，下一次请求前停下）。
  * - 附件按快照自己的组织与原对话取（C3）。
@@ -33,6 +42,11 @@ export interface RunReplayResult {
     usage: ModelUsage;
     steps: number;
     unmatched: number;
+    /**
+     * Set when a call was matched with a recording made under the release-candidate tool names
+     * (`sandbox_edit` for `Edit`, `docs__search` for `mcp__docs__search`, old parameter names).
+     */
+    legacyKeyMatch?: true;
 }
 /** 原运行（基线）的同形摘要。 */
 export type RunBaseline = Omit<RunReplayResult, "conversationId" | "outcome" | "unmatched">;
@@ -50,9 +64,14 @@ export interface EvalReplayHooks {
     beforeJudge?(evalRunId: string): Promise<void>;
 }
 export interface RunReplayDeps {
-    db: KernelDb;
+    /** The context-aware handle (reads, the eval conversation's cleanup). */
+    readonly db: KernelSchemaDb;
     eventLog: EventLog;
-    config: () => KernelConfig;
+    /** Writes the seed (the eval conversation, its events, the run) in a transaction of its own. */
+    seeder: Pick<EvalReplaySeeder, "seed">;
+    /** Writes the replayed run's terminal state. */
+    runFinish: Pick<RunFinish, "finish">;
+    config: Pick<KernelConfigService, "get">;
     log: Logger;
     registry: Pick<ProviderRegistry, "resolve">;
     media?: MediaSourceDeps;
@@ -64,15 +83,61 @@ export interface RunReplayDeps {
     /** 本次执行是否仍持有结果的认领（失去认领 → 不再发请求）；不给 = 不检查。 */
     stillClaimed?: () => Promise<boolean>;
     /**
-     * 种子事务内调用：把评测对话记到结果上（以认领为条件）。返回 false = 已失去认领，种子回滚、抛 {@link ReplayClaimLost}。
+     * 种子事务内调用（its queries join the seed's transaction through the context-aware handle）：把评测对话记到结果上
+     * （以认领为条件）。返回 false = 已失去认领，种子回滚、抛 {@link ReplayClaimLost}。
      */
-    onSeeded?: (trx: KernelDb, conversationId: string) => Promise<boolean>;
+    onSeeded?: (conversationId: string) => Promise<boolean>;
     hooks?: EvalReplayHooks;
 }
 /** 写种子时发现已失去认领或运行已不在执行：本次执行停下，不留评测对话。 */
 export declare class ReplayClaimLost extends Error {
     constructor();
 }
+/** What {@link EvalReplaySeeder.seed} writes. */
+export interface ReplaySeed {
+    evalRunId: string;
+    orgId: string;
+    agentId: string;
+    accountId: string;
+    conversationId: string;
+    runId: string;
+    /** The original run's `run_started` seq: the seeds are the events before it. */
+    runStartedSeq: number;
+    /** The conversation's `consumed_seq` when the original run started. */
+    consumed: number;
+    seeds: SnapshotEvent[];
+    /** The run's trigger seq (its boundary: the last unconsumed message, or the waking finish). */
+    trigger: number;
+    agentVersion: number;
+    model: ModelRef;
+    /** The original run's `skills_catalog` event, its payload copied as is. */
+    catalog: {
+        payload: unknown;
+    } | undefined;
+    /**
+     * The original run's `run_context` event (run context spec §8), copied as is: a candidate sees
+     * the original date, names and org, never the eval day's.
+     */
+    runContext: {
+        payload: unknown;
+    } | undefined;
+}
+/**
+ * The seed of a full-run replay (C6): the `kind = 'eval'` conversation with the events before the
+ * original run at their original seqs, the run row and its `run_started` (and the original
+ * `skills_catalog`), in one transaction of its own.
+ */
+export declare class EvalReplaySeeder {
+    private readonly db;
+    private readonly eventLog;
+    readonly txHost: TransactionHost;
+    constructor(db: KernelDb, eventLog: Pick<EventLog, "appendIn">, txHost: TransactionHost);
+    /**
+     * Writes the seed; `onSeeded` runs inside the same transaction (false = the claim is lost: the
+     * seed rolls back and {@link ReplayClaimLost} is thrown).
+     */
+    seed(p: ReplaySeed, onSeeded?: (conversationId: string) => Promise<boolean>): Promise<void>;
+}
 /** 整轮回放被停下的原因（`run_aborted.detail`）。 */
 export type StopDetail = "cancelled" | "capped" | "too_many_errors" | "claim_lost" | "shutdown";
 /** 原运行（快照）的摘要，作为整轮回放的基线。 */