@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/evals/trajectory.service.js

@@ -0,0 +1,323 @@
+import { sql } from "kysely";
+import { isAgentVisibleTo } from "../agents/agents.service.js";
+import { ApiError } from "../errors.js";
+import { audit } from "../identity/audit.js";
+import { roleAtLeast } from "../orgs/roles.js";
+import { buildTrajectory } from "./trajectory.js";
+/** 运行的状态（`runs.status`），列表按最近一次运行的状态过滤用。 */
+export const RUN_STATUSES = [
+    "queued",
+    "running",
+    "waiting",
+    "completed",
+    "failed",
+    "aborted",
+];
+/** 一次读事件的页大小。 */
+const EVENT_PAGE = 1000;
+const notFound = () => new ApiError("trajectory.not_found", "Trajectory not found");
+/**
+ * 轨迹查看器（D6、C1）：只给组织管理员（admin / owner）与 agent 负责人。负责人与角色每次实时读（R5）；
+ * 无权查看与不存在一律 `trajectory.not_found`，不泄露对话是否存在。每次查看详情写审计 `trajectory.viewed`。
+ * 企业管理员不因身份而能看（不是组织成员时 OrgGuard 已 404）。
+ *
+ * 列表按 `conversations.kind` 过滤，不传时只列 `chat`（评测回放的隐藏对话用 `kind: "eval"` 列出）。
+ * 评测对话（`kind = 'eval'`）只给组织管理员（Ruling S1-c）：里面是管理员的候选指令，非管理员的 agent 负责人
+ * 列不出、读详情 404。
+ */
+export class TrajectoryService {
+    db;
+    eventLog;
+    prices;
+    settings;
+    constructor(db, eventLog, prices, settings) {
+        this.db = db;
+        this.eventLog = eventLog;
+        this.prices = prices;
+        this.settings = settings;
+    }
+    /** 调用者当前在组织里的角色；不是成员或组织已删除 → null。 */
+    async role(orgId, accountId) {
+        const m = await this.db
+            .selectFrom("memberships")
+            .innerJoin("orgs", "orgs.id", "memberships.org_id")
+            .select("memberships.role")
+            .where("memberships.org_id", "=", orgId)
+            .where("memberships.account_id", "=", accountId)
+            .where("orgs.deleted_at", "is", null)
+            .executeTakeFirst();
+        return m?.role ?? null;
+    }
+    /** 至少是 member（guest 不能看轨迹）。 */
+    static isMember(role) {
+        return role !== null && roleAtLeast(role, "member");
+    }
+    /** 组织 admin / owner，或该 agent 当前的负责人（且至少是 member）。agent 已删除或不在本组织 → false。 */
+    async canView(orgId, accountId, agentId) {
+        const role = await this.role(orgId, accountId);
+        if (!TrajectoryService.isMember(role))
+            return false;
+        const agent = await this.db
+            .selectFrom("agents")
+            .select("owner_account_id")
+            .where("id", "=", agentId)
+            .where("org_id", "=", orgId)
+            .where("deleted_at", "is", null)
+            .executeTakeFirst();
+        if (!agent)
+            return false;
+        return roleAtLeast(role, "admin") || agent.owner_account_id === accountId;
+    }
+    /**
+     * 列出可查看的对话（admin 看全部，其余只看自己负责的 agent），按 (updated_at, id) 倒序；`kind` 缺省为 `chat`。
+     * `before` 是上一页的 `next`（对话 id）：从该对话当前的 (updated_at, id) 之后继续。
+     */
+    async list(orgId, accountId, q) {
+        const role = await this.role(orgId, accountId);
+        if (!TrajectoryService.isMember(role))
+            return { items: [], next: null };
+        const admin = roleAtLeast(role, "admin");
+        // 评测对话只给组织管理员（Ruling S1-c）
+        if (q.kind === "eval" && !admin)
+            return { items: [], next: null };
+        let query = this.db
+            .selectFrom("conversations as c")
+            .innerJoin("agents as a", "a.id", "c.agent_id")
+            .innerJoin("agent_versions as v", (j) => j.onRef("v.agent_id", "=", "a.id").onRef("v.version", "=", "a.current_version"))
+            .select([
+            "c.id",
+            "c.agent_id",
+            "c.account_id",
+            "c.title",
+            "c.updated_at",
+            sql `v.definition ->> 'name'`.as("agent_name"),
+        ])
+            .where("c.org_id", "=", orgId)
+            .where("c.kind", "=", q.kind ?? "chat")
+            .where("a.org_id", "=", orgId)
+            .where("a.deleted_at", "is", null)
+            .orderBy("c.updated_at", "desc")
+            .orderBy("c.id", "desc")
+            .limit(q.limit + 1);
+        if (!admin)
+            query = query.where("a.owner_account_id", "=", accountId);
+        if (q.agentId !== undefined)
+            query = query.where("c.agent_id", "=", q.agentId);
+        if (q.status !== undefined) {
+            const status = q.status;
+            // 与下面取 lastRun 的排序一致：最近一次运行 = created_at、id 最大的一条
+            query = query.where((eb) => sql `(
+            SELECT r.status FROM capybara.runs r WHERE r.conversation_id = ${eb.ref("c.id")}
+            ORDER BY r.created_at DESC, r.id DESC LIMIT 1
+          ) = ${status}`);
+        }
+        if (q.before !== undefined) {
+            const before = q.before;
+            query = query.where((eb) => sql `(${eb.ref("c.updated_at")}, ${eb.ref("c.id")}) < (
+            SELECT x.updated_at, x.id FROM capybara.conversations x
+            WHERE x.id = ${before} AND x.org_id = ${orgId}
+          )`);
+        }
+        const rows = await query.execute();
+        const page = rows.slice(0, q.limit);
+        const ids = page.map((r) => r.id);
+        const lastRuns = ids.length === 0
+            ? []
+            : await this.db
+                .selectFrom("runs")
+                .select(["id", "conversation_id", "status", "started_at"])
+                .distinctOn("conversation_id")
+                .where("conversation_id", "in", ids)
+                .orderBy("conversation_id")
+                .orderBy("created_at", "desc")
+                .orderBy("id", "desc")
+                .execute();
+        const byConv = new Map(lastRuns.map((r) => [r.conversation_id, r]));
+        const items = page.map((r) => {
+            const run = byConv.get(r.id);
+            return {
+                conversationId: r.id,
+                agentId: r.agent_id,
+                agentName: r.agent_name ?? "",
+                accountId: r.account_id,
+                title: r.title,
+                updatedAt: r.updated_at.toISOString(),
+                lastRun: run === undefined
+                    ? null
+                    : {
+                        runId: run.id,
+                        status: run.status,
+                        startedAt: run.started_at === null ? null : run.started_at.toISOString(),
+                    },
+            };
+        });
+        const next = rows.length > q.limit ? (page.at(-1)?.id ?? null) : null;
+        return { items, next };
+    }
+    /**
+     * 一个对话的轨迹；无权或不存在 → `trajectory.not_found`。成功时（含窗口为空）写审计 `trajectory.viewed`。
+     * 不给 `window` 时读全对话；给了只读窗口内（且已开始）的运行的事件，响应带 `next`。
+     */
+    async get(orgId, accountId, conversationId, actorTokenId, window) {
+        const conv = await this.db
+            .selectFrom("conversations")
+            .select(["id", "agent_id", "account_id", "title", "kind", "parent_conversation_id"])
+            .where("id", "=", conversationId)
+            .where("org_id", "=", orgId)
+            .executeTakeFirst();
+        // A sub-agent's trajectory is part of its parent conversation's: authorized by the parent
+        // conversation's agent, never by the dispatched agent (an org agent's owner must not read a
+        // child drawn from someone else's conversation; batch 9 review I1)
+        const authAgentId = conv?.kind === "subagent"
+            ? await this.parentAgentId(orgId, conv.parent_conversation_id)
+            : conv?.agent_id;
+        if (!conv || authAgentId === null || authAgentId === undefined)
+            throw notFound();
+        if (!(await this.canView(orgId, accountId, authAgentId)))
+            throw notFound();
+        // an org agent child also needs that agent to be visible to a non-admin viewer, as on the
+        // conversation read path (batch 9 ruling; final review m3)
+        if (conv.kind === "subagent" && conv.agent_id !== authAgentId) {
+            const role = await this.role(orgId, accountId);
+            if ((role === null || !roleAtLeast(role, "admin")) &&
+                !(await isAgentVisibleTo(this.db, conv.agent_id, accountId))) {
+                throw notFound();
+            }
+        }
+        // 评测对话只给组织管理员（Ruling S1-c）：负责人也 404
+        if (conv.kind === "eval") {
+            const role = await this.role(orgId, accountId);
+            if (role === null || !roleAtLeast(role, "admin"))
+                throw notFound();
+        }
+        const events = [];
+        let next;
+        if (window === undefined) {
+            for (let after = 0;;) {
+                const page = await this.eventLog.readAfter(conversationId, after, EVENT_PAGE);
+                events.push(...page);
+                const last = page.at(-1);
+                if (page.length < EVENT_PAGE || last === undefined)
+                    break;
+                after = last.seq;
+            }
+        }
+        else {
+            const picked = await this.windowRuns(conversationId, window);
+            next = picked.next;
+            // 运行的事件都在它的 trigger_seq 之后：从窗口里最小的 trigger_seq 开始读，不扫更早的历史
+            let after = Math.min(...picked.runs.map((r) => r.triggerSeq));
+            const ids = picked.runs.map((r) => r.id);
+            while (ids.length > 0) {
+                const page = await this.eventLog.readRunsAfter(conversationId, ids, after, EVENT_PAGE);
+                events.push(...page);
+                const last = page.at(-1);
+                if (page.length < EVENT_PAGE || last === undefined)
+                    break;
+                after = last.seq;
+            }
+        }
+        const [table, settings, parent] = await Promise.all([
+            this.prices.table(),
+            this.settings.get(),
+            conv.kind === "subagent" ? this.firstDispatch(conv.id) : undefined,
+        ]);
+        await audit(this.db, {
+            actorAccountId: accountId,
+            ...(actorTokenId !== undefined && { actorTokenId }),
+            orgId,
+            action: "trajectory.viewed",
+            target: conversationId,
+            detail: { agentId: conv.agent_id },
+        });
+        return {
+            conversation: {
+                id: conv.id,
+                agentId: conv.agent_id,
+                accountId: conv.account_id,
+                title: conv.title,
+                kind: conv.kind,
+                ...(parent !== undefined && { parent }),
+            },
+            runs: buildTrajectory(events, table),
+            currency: settings.priceCurrency,
+            ...(next !== undefined && { next }),
+        };
+    }
+    /** The parent call that first dispatched a sub-agent conversation. */
+    async firstDispatch(childId) {
+        const row = await this.db
+            .selectFrom("subagent_dispatches")
+            .select(["parent_conversation_id", "parent_run_id", "parent_call_id"])
+            .where("child_conversation_id", "=", childId)
+            .orderBy("created_at")
+            .orderBy("resumed")
+            .executeTakeFirst();
+        return row === undefined
+            ? undefined
+            : {
+                conversationId: row.parent_conversation_id,
+                runId: row.parent_run_id,
+                callId: row.parent_call_id,
+            };
+    }
+    /** The agent of a sub-agent conversation's parent conversation, or null. */
+    async parentAgentId(orgId, parentId) {
+        if (parentId === null)
+            return null;
+        const p = await this.db
+            .selectFrom("conversations")
+            .select("agent_id")
+            .where("id", "=", parentId)
+            .where("org_id", "=", orgId)
+            .executeTakeFirst();
+        return p?.agent_id ?? null;
+    }
+    /**
+     * 窗口内已开始（`started_at` 非空，与轨迹只列有 `run_started` 的运行一致）的运行，按 (created_at, id) 倒序取
+     * `limit` 次；`before` 是上一页的 `next`（运行 id）。`next`：还有更早的运行时为本页最早一次的 id。
+     * `runId` / `before` 不是本对话里已开始的运行 → `trajectory.not_found`（调用方在写审计之前调用）。
+     */
+    async windowRuns(conversationId, window) {
+        let query = this.db
+            .selectFrom("runs as r")
+            .select(["r.id", "r.trigger_seq"])
+            .where("r.conversation_id", "=", conversationId)
+            .where("r.started_at", "is not", null)
+            .orderBy("r.created_at", "desc")
+            .orderBy("r.id", "desc");
+        if ("runId" in window) {
+            query = query.where("r.id", "=", window.runId).limit(1);
+        }
+        else {
+            query = query.limit(window.limit + 1);
+            if (window.before !== undefined) {
+                const before = window.before;
+                // 游标必须是本对话里已开始的一次运行，否则 404（不写审计）
+                const cursor = await this.db
+                    .selectFrom("runs")
+                    .select("id")
+                    .where("id", "=", before)
+                    .where("conversation_id", "=", conversationId)
+                    .where("started_at", "is not", null)
+                    .executeTakeFirst();
+                if (!cursor)
+                    throw notFound();
+                query = query.where((eb) => sql `(${eb.ref("r.created_at")}, ${eb.ref("r.id")}) < (
+              SELECT x.created_at, x.id FROM capybara.runs x
+              WHERE x.id = ${before} AND x.conversation_id = ${conversationId}
+            )`);
+            }
+        }
+        const rows = await query.execute();
+        // 指定的运行不在本对话里（或还没开始）：404（不写审计）
+        if ("runId" in window && rows.length === 0)
+            throw notFound();
+        const limit = "runId" in window ? 1 : window.limit;
+        const page = rows.slice(0, limit);
+        return {
+            runs: page.map((r) => ({ id: r.id, triggerSeq: Number(r.trigger_seq) })),
+            next: rows.length > limit ? (page.at(-1)?.id ?? null) : null,
+        };
+    }
+}

package/dist/evals/worker.d.ts

@@ -0,0 +1,101 @@
+import { type Logger } from "@capybara-hq/plugin-sdk";
+import type { KernelConfig } from "../config/schema.js";
+import type { KernelDb } from "../db/kernel-schema.js";
+import { type ModelPurposesService } from "../models/purposes.js";
+import { type OrgQuota } from "../runtime/quota.js";
+import { type RunReplayDeps } from "./run-replay.js";
+import { type CaseSnapshot } from "./snapshot.js";
+import { type StepReplayDeps } from "./step-replay.js";
+/**
+ * 评测 worker（C7，只在含 `worker` 角色的进程启动）：领取 `cap-kernel-evals` 的任务，每个任务回放一个用例。
+ * - 运行不活跃（取消 / 达到上限 / 熔断 / 已结束）→ 该用例剩余结果标 `skipped`；
+ * - 组织并发配额 `capk:org-evals:<orgId>`（`evals.orgConcurrency`，成员为 jobId + 认领号）拿不到 → 延后 5000ms；
+ * - 每次请求前查 `runActive`（含组织 / agent 未删除）与认领；成功的请求在同一事务里写结果并 `consumeTokens`（越限即 `capped`），
+ *   发出后失败的尝试按估算同样 `consumeTokens`；
+ * - 可重试的错误重试 3 次，不可重试的结果标 `error`；连续 `evals.maxConsecutiveErrors` 条结果出错 → 运行 `failed`；
+ * - 结果按认领号（每次执行一个 uuid）写入：失去认领的旧执行写不进来（已发出请求的用量照样计入）；
+ * - 结果写 done 时一并写结构差异（`diff`）；开了评委时随后（运行仍活跃才）发一次评委请求，结论写进 `judge`，
+ *   用量同样 `consumeTokens`（请求失败按估算）；
+ * - 用例做完重算 `cases_done`（已全部结束的用例数，幂等），达到总数且仍在执行 → `completed`。
+ * 多副本：状态全在 PG / Redis；崩溃的任务由 BullMQ stalled 机制交给别的 worker，`running` 的结果从头重做；
+ * 处理函数外的暂时故障（PG / Redis）按任务的 attempts 指数退避重试。
+ */
+/** 拿不到组织配额时任务延后的毫秒数。 */
+export declare const EVAL_QUOTA_RETRY_MS = 5000;
+/** 可重试错误的最多重试次数。 */
+export declare const EVAL_MAX_RETRIES = 3;
+export interface EvalWorkerDeps {
+    db: KernelDb;
+    redisUrl: string;
+    /** 评测配额（`new OrgQuota(redis, EVAL_QUOTA_PREFIX)`）。 */
+    quota: OrgQuota;
+    config: () => KernelConfig;
+    log: Logger;
+    replay: StepReplayDeps;
+    /** 整轮回放（mode = run）的依赖；认领相关的两项由 worker 按每次执行补上。 */
+    runReplay: Omit<RunReplayDeps, "stillClaimed" | "onSeeded">;
+    /** 评委（用途 `eval`，D3）：开了评委的运行每条 done 的结果发一次评委请求。 */
+    purposes: Pick<ModelPurposesService, "chat">;
+    /** 测试用：覆盖拿不到配额时的延后时长。 */
+    quotaRetryMs?: number;
+}
+export declare class EvalWorker {
+    private readonly d;
+    private worker;
+    private conn;
+    /** 本进程在执行的任务：停机时中止，任务回到队列。 */
+    private readonly active;
+    private closing;
+    constructor(d: EvalWorkerDeps);
+    start(): void;
+    /** 停止领取；中止本进程在执行的任务（回到队列由别的 worker 接着做），在 budgetMs 内等它们退出。 */
+    close(budgetMs: number): Promise<void>;
+    private process;
+    /** 单步回放每次请求前：运行仍在执行（{@link runActive}）且结果仍由本次执行认领。 */
+    private mayRequest;
+    /**
+     * 一条结果出错：只在仍由本次执行认领时写入，然后计入熔断；熔断 → 该用例剩下的结果 skipped（其他用例的任务
+     * 看到运行不活跃也会跳过）。返回 true 表示本用例应停下（熔断或失去认领）。
+     */
+    private resultError;
+    /** 回放一个用例的全部未结束结果（按调用顺序）。 */
+    private runCase;
+    /**
+     * 补做评委：结果已 done、`judge_status = 'pending'`（上一次执行在评委之前崩溃 / 停机）。按本次执行的认领号重新认领
+     * （条件仍是 done + pending），由结果里存的基线 / 候选得出评委输入；候选请求不重发。
+     */
+    private judgeLeftover;
+    /**
+     * 评委（结果已写 done、`judge_status = 'pending'` 之后）：
+     * - 运行不再活跃（取消 / 越限 / 熔断）→ 不发请求，评委记 `skipped`；
+     * - 先按用量（失败按估算）`consumeTokens`，不以认领为条件（请求确实发出了）；再在仍由本次执行认领、且仍是 pending
+     *   时写结论（`judge_status` done / error）。写之前出 DB 错误按崩溃处理：重试时补评委（会再发一次评委请求）；
+     * - 停机中止：已发出的评委请求按估算计费，任务回到队列，接手的执行补评委。
+     * 供应商插件在评委请求途中被撤销（`run.provider_unavailable`）→ 按估算计费后重试，至多 {@link EVAL_MAX_RETRIES} 次；
+     * 评委失败不计入熔断（结果本身是 done）。返回 `capped` 表示评委的用量让运行达到上限。
+     */
+    private judgeResult;
+    /**
+     * 整轮回放一个用例（一条结果）。认领结果后先删掉上一次执行留下的评测对话（崩溃重跑从头开始，C6），
+     * 再 {@link replayRun}；计费在每次模型请求上完成（含失败与失去认领的请求）。
+     * - 运行不再活跃（取消 / 越限 / 熔断）→ 结果 skipped；失去认领 → 停下，不写；
+     * - 回放出错 → 结果 error 并计入熔断；停机 → 任务回到队列，接手的执行从头重跑。
+     */
+    private runWhole;
+    /**
+     * 该用例剩下的结果标 skipped。整轮回放留下的评测对话里若还有没结束的运行（上一次执行崩溃在回放中途），删掉它，
+     * 不留永远「执行中」的评测对话；已正常收尾（`run_aborted` / `run_finished`）的保留，轨迹里能看到停在哪里。
+     * 清理失败只记日志，不挡住标 skipped（残留的随评测集或组织删除清掉）。
+     */
+    private skipCase;
+    /** BullMQ 判定任务失败（stalled 次数超限或重试用尽）：剩下的结果记 error，再重算进度。 */
+    private failCase;
+}
+/**
+ * A plugin template's child case: the template's current version, which a new dispatch would use
+ * (a template removed since gets version 0, so it always differs). Undefined for other cases.
+ */
+export declare function currentTemplate(db: KernelDb, s: CaseSnapshot): Promise<{
+    id: string;
+    version: number;
+} | undefined>;