@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.1

package/dist/evals/run-replay.js

@@ -0,0 +1,502 @@
+import { randomUUID } from "node:crypto";
+import { ModelError, } from "@capybara-hq/plugin-sdk";
+import { sql } from "kysely";
+import { normalizeStoredDefinition } from "../agents/definition.js";
+import { ApiError } from "../errors.js";
+import { estimateTokens } from "../runtime/context/tokens.js";
+import { RunError } from "../runtime/model-call.js";
+import { finishRun } from "../runtime/run-finish.js";
+import { recordedSkillForks } from "../runtime/subagents/skill-fork.js";
+import { skillsCatalogIdemKey } from "../skills/catalog.js";
+import { ACTIVE_EVAL_STATUSES, addUsage, consumeTokens, TOO_MANY_ERRORS, usageTokens, } from "./budget.js";
+import { RecordedToolPipeline, recordedCalls } from "./recorded-tools.js";
+/** 写种子时发现已失去认领或运行已不在执行：本次执行停下，不留评测对话。 */
+export class ReplayClaimLost extends Error {
+    constructor() {
+        super("the eval result is no longer claimed by this execution");
+        this.name = "ReplayClaimLost";
+    }
+}
+/** 运行不再执行或失去认领时的原因；还能继续为 null。 */
+async function stopReason(db, evalRunId, stillClaimed) {
+    const r = await db
+        .selectFrom("eval_runs as r")
+        .innerJoin("eval_sets as s", "s.id", "r.set_id")
+        .innerJoin("agents as a", "a.id", "s.agent_id")
+        .innerJoin("orgs as o", "o.id", "r.org_id")
+        .select([
+        "r.status",
+        "r.error",
+        sql `a.deleted_at IS NOT NULL OR o.deleted_at IS NOT NULL`.as("gone"),
+    ])
+        .where("r.id", "=", evalRunId)
+        .executeTakeFirst();
+    // 运行行不在，或组织 / agent 已删除（删除时会显式取消运行，这里兜底）
+    if (!r || r.gone)
+        return "cancelled";
+    if (r.status === "capped")
+        return "capped";
+    if (r.status === "cancelled")
+        return "cancelled";
+    if (r.status === "failed") {
+        return r.error?.code === TOO_MANY_ERRORS.code
+            ? "too_many_errors"
+            : "cancelled";
+    }
+    if (!ACTIVE_EVAL_STATUSES.includes(r.status))
+        return "cancelled";
+    if (stillClaimed && !(await stillClaimed()))
+        return "claim_lost";
+    return null;
+}
+/** 种子插入的分批大小。 */
+const SEED_BATCH = 500;
+const READ_PAGE = 1000;
+const ZERO = {
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+};
+/** 一次运行的工具调用序列、最后的回复、停止原因、用量与步数（基线与候选同一算法）。 */
+function summarize(events, runId) {
+    const toolCalls = [];
+    let usage = ZERO;
+    let last = null;
+    const steps = new Set();
+    for (const e of events) {
+        if (e.runId !== runId)
+            continue;
+        if (e.type === "assistant_message") {
+            const p = e.payload;
+            steps.add(p.step);
+            for (const b of p.content) {
+                if (b.type === "tool_call")
+                    toolCalls.push({ name: b.name, input: b.input });
+            }
+            last = { content: p.content, stopReason: p.stopReason };
+        }
+        else if (e.type === "usage") {
+            const u = e.payload;
+            usage = addUsage(usage, {
+                inputTokens: u.inputTokens ?? 0,
+                outputTokens: u.outputTokens ?? 0,
+                cacheReadTokens: u.cacheReadTokens ?? 0,
+                cacheWriteTokens: u.cacheWriteTokens ?? 0,
+            });
+        }
+    }
+    return {
+        toolCalls,
+        finalContent: last?.content ?? [],
+        stopReason: last === null || last.stopReason === "aborted" ? null : last.stopReason,
+        usage,
+        steps: steps.size,
+    };
+}
+/** 原运行（快照）的摘要，作为整轮回放的基线。 */
+export function runBaseline(s) {
+    return summarize(s.events, s.runId);
+}
+/**
+ * 删掉一个评测对话（崩溃重跑前清掉上一次的残留）：先删它各次运行的 checkpoint，再删对话（事件、运行随之级联删除）。
+ * 只删 `kind = 'eval'` 的对话。
+ */
+export async function deleteEvalConversation(d, conversationId) {
+    await d.hooks?.beforeDeleteConversation?.(conversationId);
+    const runs = await d.db
+        .selectFrom("runs")
+        .innerJoin("conversations", "conversations.id", "runs.conversation_id")
+        .select("runs.id")
+        .where("runs.conversation_id", "=", conversationId)
+        .where("conversations.kind", "=", "eval")
+        .execute();
+    for (const r of runs)
+        await d.checkpoints?.deleteThread(r.id);
+    await d.db
+        .deleteFrom("conversations")
+        .where("id", "=", conversationId)
+        .where("kind", "=", "eval")
+        .execute();
+}
+/**
+ * 原运行每步开始时吸收到的位置（第一次尝试的 `absorbedUpToSeq`；较早的事件没有时取该 model_call 的 seq）。
+ */
+function absorbedByStep(s) {
+    const out = new Map();
+    for (const e of s.events) {
+        if (e.runId !== s.runId || e.type !== "model_call")
+            continue;
+        const p = e.payload;
+        if (out.has(p.step))
+            continue;
+        out.set(p.step, p.request?.absorbedUpToSeq ?? e.seq);
+    }
+    return out;
+}
+const readAll = async (eventLog, conversationId) => {
+    const out = [];
+    let after = 0;
+    for (;;) {
+        const page = await eventLog.readAfter(conversationId, after, READ_PAGE);
+        out.push(...page);
+        const last = page.at(-1);
+        if (page.length < READ_PAGE || !last)
+            return out;
+        after = last.seq;
+    }
+};
+/** 整轮回放一个用例。见文件头注释。 */
+export async function replayRun(d, s, candidate, evalRunId, signal) {
+    const { db, eventLog } = d;
+    const start = s.events.find((e) => e.runId === s.runId && e.type === "run_started");
+    if (!start) {
+        throw new ApiError("eval.call_not_found", "The snapshot does not contain the run's start");
+    }
+    const run = await db
+        .selectFrom("eval_runs")
+        .select("created_by")
+        .where("id", "=", evalRunId)
+        .executeTakeFirstOrThrow();
+    const accountId = run.created_by;
+    const R = start.seq;
+    const seeds = s.events.filter((e) => e.seq < R);
+    // 原运行开始时会话的 consumed_seq：此前最近一次正常结束记下的位置（失败 / 中止的终态不带位置，取不到时为 0）
+    let consumed = 0;
+    for (const e of seeds) {
+        if (e.type === "run_finished") {
+            consumed = Math.max(consumed, e.payload.consumedUpToSeq);
+        }
+    }
+    const trigger = seeds.findLast((e) => e.type === "user_message")?.seq ?? 0;
+    const catalog = s.events.find((e) => e.runId === s.runId && e.type === "skills_catalog" && e.seq > R);
+    // a sub-agent's case (spec §15.3) runs the effective definition frozen in its subagent_started
+    // (the request renders that event's preamble and instructions; its tools come from it too), with
+    // only the candidate's model and params, and the logged step limit
+    const subagent = s.events.find((e) => e.type === "subagent_started")?.payload;
+    const agent = subagent === undefined
+        ? candidate.definition
+        : {
+            ...normalizeStoredDefinition(s.subagent?.definition ?? subagent.definition),
+            model: candidate.model,
+            modelParams: candidate.params,
+        };
+    const conversationId = randomUUID();
+    const runId = randomUUID();
+    await db.transaction().execute(async (t) => {
+        // 运行行加共享锁并复核仍在执行：与取消 / 删除评测集（UPDATE 持行锁）互斥，删除之后不会再长出评测对话
+        const live = await t
+            .selectFrom("eval_runs")
+            .select("id")
+            .where("id", "=", evalRunId)
+            .where("status", "in", [...ACTIVE_EVAL_STATUSES])
+            .forShare()
+            .executeTakeFirst();
+        if (!live)
+            throw new ReplayClaimLost();
+        await t
+            .insertInto("conversations")
+            .values({
+            id: conversationId,
+            org_id: s.orgId,
+            agent_id: s.agentId,
+            account_id: accountId,
+            last_seq: R - 1,
+            consumed_seq: consumed,
+            kind: "eval",
+            eval_run_id: evalRunId,
+        })
+            .execute();
+        for (let i = 0; i < seeds.length; i += SEED_BATCH) {
+            await t
+                .insertInto("conversation_events")
+                .values(seeds.slice(i, i + SEED_BATCH).map((e) => ({
+                conversation_id: conversationId,
+                seq: e.seq,
+                type: e.type,
+                run_id: e.runId,
+                idem_key: null,
+                payload: JSON.stringify(e.payload),
+                created_at: new Date(e.createdAt),
+            })))
+                .execute();
+        }
+        await t
+            .insertInto("runs")
+            .values({
+            id: runId,
+            conversation_id: conversationId,
+            status: "running",
+            agent_version: candidate.agentVersion,
+            trigger_account_id: accountId,
+            trigger_seq: trigger,
+            started_at: new Date(),
+        })
+            .execute();
+        await eventLog.append(conversationId, [
+            {
+                type: "run_started",
+                runId,
+                idemKey: `run_started:${runId}`,
+                payload: {
+                    agentVersion: candidate.agentVersion,
+                    model: agent.model,
+                    triggerAccountId: accountId,
+                },
+            },
+            ...(catalog
+                ? [
+                    {
+                        type: "skills_catalog",
+                        runId,
+                        idemKey: skillsCatalogIdemKey(runId),
+                        payload: catalog.payload,
+                    },
+                ]
+                : []),
+        ], t);
+        if (d.onSeeded && !(await d.onSeeded(t, conversationId)))
+            throw new ReplayClaimLost();
+    });
+    await d.hooks?.afterRunSeeded?.(evalRunId, conversationId);
+    // 停止：取消 / 越限 / 熔断 / 失去认领 / 截止时间 → 中止 loop（进行中的请求立即停下，loop 以 aborted 返回）
+    const stopper = new AbortController();
+    const loopSignal = AbortSignal.any([signal, stopper.signal]);
+    /** 为什么停下（写进 `run_aborted.detail`，轨迹据此说明原因）。 */
+    let stopped;
+    /**
+     * 检查与计费过程中的意外错误（PG 暂时故障等）：记下并中止 loop，loop 返回后按崩溃抛出
+     * （交给 BullMQ 重试、从头重跑），不让它在模型调用里被当成供应商错误、走降级梯子。
+     */
+    let crash;
+    const crashed = (err) => {
+        crash ??= err;
+        stopper.abort();
+    };
+    /**
+     * 还能发下一次请求吗。取消只在请求之间生效（与单步回放一致）：进行中的请求做完，之后不再发。
+     */
+    const allowed = async () => {
+        if (loopSignal.aborted)
+            return false;
+        await d.hooks?.beforeCheck?.(evalRunId);
+        const why = await stopReason(db, evalRunId, d.stillClaimed);
+        if (why === null)
+            return true;
+        stopped ??= why;
+        return false;
+    };
+    /** 同 {@link allowed}，但出错时记为崩溃（返回 false）。 */
+    const allowedSafe = async () => {
+        try {
+            return await allowed();
+        }
+        catch (err) {
+            crashed(err);
+            return false;
+        }
+    };
+    /** 进行中的计费（提前结束的流在 finally 里计费，不阻塞调用方）；返回前等它们写完。 */
+    const charges = new Set();
+    const charge = (n) => {
+        // 计费写不进去也按崩溃处理：不能让请求在预算之外继续发
+        const p = (async () => {
+            await d.hooks?.beforeCharge?.(evalRunId);
+            await consumeTokens(db, evalRunId, n);
+        })().then(() => { }, crashed);
+        charges.add(p);
+        void p.finally(() => charges.delete(p));
+        return p;
+    };
+    const meter = (provider) => {
+        const metered = Object.create(provider);
+        metered.stream = (req, opts) => (async function* () {
+            if (!(await allowedSafe())) {
+                stopper.abort();
+                throw new ModelError("aborted", "The eval run stopped before this request");
+            }
+            let chars = 0;
+            let charged = false;
+            try {
+                for await (const ev of provider.stream(req, opts)) {
+                    if (ev.type === "message_complete") {
+                        charged = true;
+                        await charge(usageTokens(ev.usage));
+                    }
+                    else {
+                        chars += ev.text.length;
+                    }
+                    yield ev;
+                }
+            }
+            finally {
+                // 请求已发出但没有正常收完（出错、超时、中止）：按估算计费（与单步回放一致），计完再把错误交给 loop——
+                // loop 的重试 / 兜底请求在下一次检查前就能看到这笔用量（越限即停）。请求里的媒体块已解析成字节，
+                // `estimateTokens` 逐块计入（图片 1600、文档按大小，与构建请求时的 mediaTokens 同一套估算）
+                if (!charged) {
+                    await charge(estimateTokens(req.messages, req.system, req.tools) + Math.ceil(chars / 4));
+                }
+            }
+        })();
+        return metered;
+    };
+    const registry = {
+        resolve: async (ref) => {
+            const r = await d.registry.resolve(ref);
+            return { ...r, provider: meter(r.provider) };
+        },
+    };
+    const media = d.media && {
+        info: (_org, _conv, ids) => d.media.info(s.orgId, s.conversationId, ids),
+        fetch: (_org, _conv, ids, caps) => d.media.fetch(s.orgId, s.conversationId, ids, caps),
+    };
+    const pipeline = new RecordedToolPipeline({
+        eventLog,
+        recordings: recordedCalls(s),
+        // 出错直接抛出：tools 节点的异常让 loop 抛出，按崩溃处理
+        beforeStep: async () => {
+            if (await allowed())
+                return "continue";
+            stopper.abort();
+            return "stop";
+        },
+    });
+    // 插话：原运行开始之后到达、且被原运行某一步吸收的 user_message
+    const absorbed = absorbedByStep(s);
+    const stepsInOrder = [...absorbed.keys()].sort((a, b) => a - b);
+    /** 第 k 步（含）之前原运行吸收到的最大位置；原运行没有模型调用时不设限。 */
+    const capAt = (k) => {
+        const first = stepsInOrder[0];
+        if (first === undefined)
+            return Number.POSITIVE_INFINITY;
+        let cap = absorbed.get(first);
+        for (const step of stepsInOrder)
+            if (step <= k)
+                cap = Math.max(cap, absorbed.get(step));
+        return cap;
+    };
+    const interjections = s.events
+        .filter((e) => e.type === "user_message" && e.seq > R)
+        .flatMap((e) => {
+        const step = stepsInOrder.find((k) => absorbed.get(k) >= e.seq);
+        return step === undefined ? [] : [{ event: e, step }];
+    });
+    /** 已追加的插话：原 seq → 评测对话里的 seq。 */
+    const fed = new Map();
+    let prepared = 0;
+    const pendingUserMessages = async (afterSeq) => {
+        prepared++;
+        // loop 的步骤从 1 起（与运行时相同），每步的 prepare 调用一次：第 n 次调用即第 n 步
+        const k = prepared;
+        for (const x of interjections) {
+            if (x.step > k || fed.has(x.event.seq))
+                continue;
+            const [w] = await eventLog.append(conversationId, [
+                {
+                    type: "user_message",
+                    idemKey: `replay_interjection:${runId}:${x.event.seq}`,
+                    payload: x.event.payload,
+                },
+            ]);
+            if (w)
+                fed.set(x.event.seq, w.seq);
+        }
+        const cap = capAt(k);
+        const fedSeqs = new Set(fed.values());
+        const out = [];
+        for (const e of await readAll(eventLog, conversationId)) {
+            if (e.type !== "user_message" || e.seq <= afterSeq)
+                continue;
+            if (fedSeqs.has(e.seq) || (e.seq < R && e.seq <= cap))
+                out.push(e);
+        }
+        return out;
+    };
+    const runtime = d.config().runtime;
+    const deadline = Date.now() + runtime.maxRunSeconds * 1000;
+    let deadlineHit = false;
+    const timer = setTimeout(() => {
+        deadlineHit = true;
+        stopper.abort();
+    }, Math.max(0, deadline - Date.now()));
+    timer.unref();
+    // a fork invocation's report is the recorded one: a replay never dispatches a sub-agent (A5)
+    const skillForks = recordedSkillForks(eventLog, s.events.filter((e) => e.runId === s.runId));
+    const loop = d.loop({ toolPipeline: pipeline, registry, skillForks, ...(media && { media }) });
+    let outcome;
+    try {
+        outcome = await loop.run({
+            orgId: s.orgId,
+            conversationId,
+            runId,
+            agentId: s.agentId,
+            triggerAccountId: accountId,
+            agent,
+            agentVersion: candidate.agentVersion,
+            signal: loopSignal,
+            shutdown: new AbortController().signal,
+            sink: () => { },
+            pendingUserMessages,
+            limits: {
+                maxSteps: subagent === undefined ? runtime.maxSteps : Math.min(subagent.maxSteps, runtime.maxSteps),
+                deadline,
+            },
+            // replays never fire plugin hooks (the loop deps omit them too, P-18)
+            conversationKind: "eval",
+        });
+    }
+    catch (e) {
+        const err = e instanceof RunError ? e : new RunError("run.internal", "The run failed unexpectedly");
+        await finishRun(db, eventLog, {
+            conversationId,
+            runId,
+            status: "failed",
+            consumed: null,
+            event: { type: "run_failed", payload: { error: { code: err.code, message: err.message } } },
+        }).catch((x) => d.log.warn("cannot finish eval replay run", { runId, err: x }));
+        await Promise.allSettled([...charges]);
+        throw crash ?? e;
+    }
+    finally {
+        clearTimeout(timer);
+        await d.checkpoints
+            ?.deleteThread(runId)
+            .catch((err) => d.log.warn("eval replay checkpoint delete failed", { runId, err }));
+    }
+    await Promise.allSettled([...charges]);
+    if (crash !== undefined)
+        throw crash;
+    if (outcome.kind === "aborted" && deadlineHit && !signal.aborted && stopped === undefined) {
+        outcome = { kind: "max_duration", consumedUpToSeq: 0 };
+    }
+    if (outcome.kind === "aborted" && stopped === undefined && signal.aborted)
+        stopped = "shutdown";
+    const finalConsumed = Math.max(0, ...fed.values(), trigger);
+    await finishRun(db, eventLog, {
+        conversationId,
+        runId,
+        status: outcome.kind === "aborted" ? "aborted" : "completed",
+        consumed: finalConsumed,
+        event: outcome.kind === "aborted"
+            ? {
+                type: "run_aborted",
+                payload: {
+                    byAccountId: null,
+                    reason: "eval_stopped",
+                    ...(stopped !== undefined && { detail: stopped }),
+                },
+            }
+            : {
+                type: "run_finished",
+                payload: {
+                    reason: outcome.kind === "max_steps" || outcome.kind === "max_duration"
+                        ? outcome.kind
+                        : "completed",
+                    consumedUpToSeq: "consumedUpToSeq" in outcome ? Math.max(outcome.consumedUpToSeq, trigger) : trigger,
+                },
+            },
+    });
+    await Promise.allSettled([...charges]);
+    const summary = summarize(await readAll(eventLog, conversationId), runId);
+    return { conversationId, outcome: outcome.kind, ...summary, unmatched: pipeline.unmatched() };
+}

package/dist/evals/runs.controller.d.ts

@@ -0,0 +1,100 @@
+import { z } from "zod";
+import { type Principal } from "../identity/principal.js";
+import { type OrgContext } from "../orgs/org.guard.js";
+import type { EvalEstimate, EvalResultLight, EvalResultView, EvalRunsService, EvalRunView, EvalTokenCaps } from "./runs.service.js";
+/** `tokenCap` 只校验类型，范围由服务判定（`eval.token_cap_invalid`）。 */
+declare const EstimateBody: z.ZodObject<{
+    mode: z.ZodEnum<{
+        run: "run";
+        step: "step";
+    }>;
+    candidate: z.ZodObject<{
+        agentVersion: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strict>>;
+        modelParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+        instructions: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    judge: z.ZodDefault<z.ZodBoolean>;
+    tokenCap: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+declare const StartBody: z.ZodObject<{
+    mode: z.ZodEnum<{
+        run: "run";
+        step: "step";
+    }>;
+    candidate: z.ZodObject<{
+        agentVersion: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strict>>;
+        modelParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+        instructions: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    judge: z.ZodDefault<z.ZodBoolean>;
+    tokenCap: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+/**
+ * `view=light`：不含内容块（结果表与轮询用，响应带 `asOf`）；缺省 `full`。`updatedSince`（ISO 时间）只取之后有变化的，
+ * 轮询时传上一次的 `asOf`。
+ */
+declare const ResultsQuery: z.ZodObject<{
+    after: z.ZodOptional<z.ZodString>;
+    limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    view: z.ZodDefault<z.ZodEnum<{
+        full: "full";
+        light: "light";
+    }>>;
+    updatedSince: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+declare const LabelBody: z.ZodObject<{
+    label: z.ZodNullable<z.ZodEnum<{
+        fail: "fail";
+        pass: "pass";
+    }>>;
+    note: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+/**
+ * 评测运行（`/api/v1/orgs/:orgId/eval-sets/:setId/runs`、`/api/v1/orgs/:orgId/eval-runs/:runId[/results[/:resultId]]`）与
+ * 结果的人工标注（`/api/v1/orgs/:orgId/eval-results/:resultId/label`）：
+ * 全部 `evals.manage`（admin 起、只给人）。
+ */
+export declare class EvalRunsController {
+    private readonly runs;
+    constructor(runs: EvalRunsService);
+    estimate(principal: Principal, org: OrgContext, setId: string, body: z.infer<typeof EstimateBody>): Promise<{
+        estimate: EvalEstimate;
+    } & EvalTokenCaps>;
+    start(principal: Principal, org: OrgContext, setId: string, body: z.infer<typeof StartBody>): Promise<{
+        run: EvalRunView;
+    }>;
+    list(principal: Principal, org: OrgContext, setId: string): Promise<{
+        runs: EvalRunView[];
+    }>;
+    get(principal: Principal, org: OrgContext, runId: string): Promise<{
+        run: EvalRunView;
+    }>;
+    results(principal: Principal, org: OrgContext, runId: string, q: z.infer<typeof ResultsQuery>): Promise<{
+        results: EvalResultView[];
+        next: string | null;
+    } | {
+        results: EvalResultLight[];
+        next: string | null;
+        asOf: string;
+    }>;
+    /** 一条结果的完整行（查看器按需读取）。 */
+    result(principal: Principal, org: OrgContext, runId: string, resultId: string): Promise<{
+        result: EvalResultView;
+    }>;
+    cancel(principal: Principal, org: OrgContext, runId: string): Promise<{
+        run: EvalRunView;
+    }>;
+    /** 人工标注一条结果：`label` 为 pass / fail，或 null 清除（备注随之清除）。 */
+    label(principal: Principal, org: OrgContext, resultId: string, body: z.infer<typeof LabelBody>): Promise<{
+        result: EvalResultView;
+    }>;
+}
+export {};