@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/evals/sets.controller.d.ts

@@ -0,0 +1,61 @@
+import { z } from "zod";
+import { type Principal } from "../identity/principal.js";
+import { type OrgContext } from "../orgs/org.guard.js";
+import type { EvalCaseDetail, EvalCaseView, EvalSetDetail, EvalSetsService, EvalSetView } from "./sets.service.js";
+declare const CreateBody: z.ZodObject<{
+    agentId: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    rubric: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+declare const PatchBody: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    rubric: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+}, z.core.$strict>;
+declare const AddCaseBody: z.ZodObject<{
+    conversationId: z.ZodString;
+    runId: z.ZodString;
+}, z.core.$strict>;
+declare const SampleBody: z.ZodObject<{
+    from: z.ZodOptional<z.ZodISODateTime>;
+    to: z.ZodOptional<z.ZodISODateTime>;
+    count: z.ZodNumber;
+    status: z.ZodOptional<z.ZodEnum<{
+        any: "any";
+        completed: "completed";
+        failed: "failed";
+    }>>;
+    minToolCalls: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+/**
+ * 评测集与用例（`/api/v1/orgs/:orgId/eval-sets`）：全部 `evals.manage`（admin 起、只给人）。
+ * 用例详情返回快照还原的轨迹，每次写审计 `trajectory.viewed`。
+ */
+export declare class EvalSetsController {
+    private readonly sets;
+    constructor(sets: EvalSetsService);
+    create(principal: Principal, org: OrgContext, body: z.infer<typeof CreateBody>): Promise<{
+        set: EvalSetView;
+    }>;
+    list(principal: Principal, org: OrgContext): Promise<{
+        sets: EvalSetView[];
+    }>;
+    get(principal: Principal, org: OrgContext, setId: string): Promise<{
+        set: EvalSetDetail;
+    }>;
+    update(principal: Principal, org: OrgContext, setId: string, body: z.infer<typeof PatchBody>): Promise<{
+        set: EvalSetView;
+    }>;
+    remove(principal: Principal, org: OrgContext, setId: string): Promise<void>;
+    addCase(principal: Principal, org: OrgContext, setId: string, body: z.infer<typeof AddCaseBody>): Promise<{
+        case: EvalCaseView;
+    }>;
+    sample(principal: Principal, org: OrgContext, setId: string, body: z.infer<typeof SampleBody>): Promise<{
+        added: number;
+        skipped: number;
+    }>;
+    getCase(principal: Principal, org: OrgContext, setId: string, caseId: string): Promise<EvalCaseDetail>;
+    removeCase(principal: Principal, org: OrgContext, setId: string, caseId: string): Promise<void>;
+}
+export {};

package/dist/evals/sets.controller.js

@@ -0,0 +1,183 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { Body, Controller, Delete, Get, HttpCode, Inject, Param, Patch, Post, } from "@nestjs/common";
+import { z } from "zod";
+import { EVAL_SETS } from "../http/tokens.js";
+import { ZodPipe } from "../http/zod.pipe.js";
+import { CurrentPrincipal, patTokenId, requireUser, } from "../identity/principal.js";
+import { CurrentOrg, OrgAction } from "../orgs/org.guard.js";
+const NAME = z.string().trim().min(1).max(200);
+const DESCRIPTION = z.string().max(2000);
+const RUBRIC = z.string().min(1).max(20_000);
+const CreateBody = z
+    .object({
+    agentId: z.string().min(1).max(100),
+    name: NAME,
+    description: DESCRIPTION.optional(),
+    rubric: RUBRIC.optional(),
+})
+    .strict();
+const PatchBody = z
+    .object({
+    name: NAME.optional(),
+    description: DESCRIPTION.optional(),
+    rubric: RUBRIC.nullable().optional(),
+})
+    .strict();
+const AddCaseBody = z
+    .object({ conversationId: z.string().min(1).max(100), runId: z.string().min(1).max(100) })
+    .strict();
+const SampleBody = z
+    .object({
+    from: z.iso.datetime({ offset: true }).optional(),
+    to: z.iso.datetime({ offset: true }).optional(),
+    count: z.number().int().min(1).max(50),
+    status: z.enum(["completed", "failed", "any"]).optional(),
+    minToolCalls: z.number().int().min(0).max(1000).optional(),
+})
+    .strict();
+/**
+ * 评测集与用例（`/api/v1/orgs/:orgId/eval-sets`）：全部 `evals.manage`（admin 起、只给人）。
+ * 用例详情返回快照还原的轨迹，每次写审计 `trajectory.viewed`。
+ */
+let EvalSetsController = class EvalSetsController {
+    sets;
+    constructor(sets) {
+        this.sets = sets;
+    }
+    async create(principal, org, body) {
+        const p = requireUser(principal);
+        const set = await this.sets.create(org.orgId, p.account.id, {
+            agentId: body.agentId,
+            name: body.name,
+            ...(body.description !== undefined && { description: body.description }),
+            ...(body.rubric !== undefined && { rubric: body.rubric }),
+        }, patTokenId(p));
+        return { set };
+    }
+    async list(principal, org) {
+        requireUser(principal);
+        return { sets: await this.sets.list(org.orgId) };
+    }
+    async get(principal, org, setId) {
+        requireUser(principal);
+        return { set: await this.sets.get(org.orgId, setId) };
+    }
+    async update(principal, org, setId, body) {
+        const p = requireUser(principal);
+        const set = await this.sets.update(org.orgId, p.account.id, setId, {
+            ...(body.name !== undefined && { name: body.name }),
+            ...(body.description !== undefined && { description: body.description }),
+            ...(body.rubric !== undefined && { rubric: body.rubric }),
+        }, patTokenId(p));
+        return { set };
+    }
+    async remove(principal, org, setId) {
+        const p = requireUser(principal);
+        await this.sets.remove(org.orgId, p.account.id, setId, patTokenId(p));
+    }
+    async addCase(principal, org, setId, body) {
+        const p = requireUser(principal);
+        return { case: await this.sets.addCase(org.orgId, p.account.id, setId, body, patTokenId(p)) };
+    }
+    async sample(principal, org, setId, body) {
+        const p = requireUser(principal);
+        return this.sets.sample(org.orgId, p.account.id, setId, {
+            count: body.count,
+            ...(body.from !== undefined && { from: body.from }),
+            ...(body.to !== undefined && { to: body.to }),
+            ...(body.status !== undefined && { status: body.status }),
+            ...(body.minToolCalls !== undefined && { minToolCalls: body.minToolCalls }),
+        }, patTokenId(p));
+    }
+    async getCase(principal, org, setId, caseId) {
+        const p = requireUser(principal);
+        return this.sets.getCase(org.orgId, p.account.id, setId, caseId, patTokenId(p));
+    }
+    async removeCase(principal, org, setId, caseId) {
+        const p = requireUser(principal);
+        await this.sets.removeCase(org.orgId, p.account.id, setId, caseId, patTokenId(p));
+    }
+};
+__decorate([
+    Post(),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Body(new ZodPipe(CreateBody)))
+], EvalSetsController.prototype, "create", null);
+__decorate([
+    Get(),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg())
+], EvalSetsController.prototype, "list", null);
+__decorate([
+    Get(":setId"),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId"))
+], EvalSetsController.prototype, "get", null);
+__decorate([
+    Patch(":setId"),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId")),
+    __param(3, Body(new ZodPipe(PatchBody)))
+], EvalSetsController.prototype, "update", null);
+__decorate([
+    Delete(":setId"),
+    HttpCode(204),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId"))
+], EvalSetsController.prototype, "remove", null);
+__decorate([
+    Post(":setId/cases"),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId")),
+    __param(3, Body(new ZodPipe(AddCaseBody)))
+], EvalSetsController.prototype, "addCase", null);
+__decorate([
+    Post(":setId/cases/sample"),
+    HttpCode(200),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId")),
+    __param(3, Body(new ZodPipe(SampleBody)))
+], EvalSetsController.prototype, "sample", null);
+__decorate([
+    Get(":setId/cases/:caseId"),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId")),
+    __param(3, Param("caseId"))
+], EvalSetsController.prototype, "getCase", null);
+__decorate([
+    Delete(":setId/cases/:caseId"),
+    HttpCode(204),
+    OrgAction("evals.manage"),
+    __param(0, CurrentPrincipal()),
+    __param(1, CurrentOrg()),
+    __param(2, Param("setId")),
+    __param(3, Param("caseId"))
+], EvalSetsController.prototype, "removeCase", null);
+EvalSetsController = __decorate([
+    Controller("api/v1/orgs/:orgId/eval-sets"),
+    __param(0, Inject(EVAL_SETS))
+], EvalSetsController);
+export { EvalSetsController };

package/dist/evals/sets.service.d.ts

@@ -0,0 +1,128 @@
+import type { KernelConfig } from "../config/schema.js";
+import type { KernelDb } from "../db/kernel-schema.js";
+import type { SettingsService } from "../identity/settings.js";
+import type { PricesService } from "../models/prices.js";
+import { type CaseSnapshot } from "./snapshot.js";
+import { type TrajectoryRun } from "./trajectory.js";
+export interface EvalSetView {
+    id: string;
+    agentId: string;
+    name: string;
+    description: string;
+    rubric: string | null;
+    caseCount: number;
+    createdAt: string;
+}
+export interface EvalCaseView {
+    id: string;
+    sourceConversationId: string;
+    sourceRunId: string;
+    agentVersion: number;
+    title: string;
+    replayableSteps: number;
+    createdAt: string;
+}
+/** 评测集详情：集合本身与全部用例（按加入顺序；不含快照）。 */
+export interface EvalSetDetail extends EvalSetView {
+    cases: EvalCaseView[];
+}
+/** 用例详情：快照还原出的轨迹（快照里的全部运行，含该运行之前的历史）。 */
+export interface EvalCaseDetail {
+    case: EvalCaseView;
+    trajectory: TrajectoryRun[];
+    /** 成本的币种（部署设置 `priceCurrency`）。 */
+    currency: string;
+    /** A sub-agent's case: the parent call that dispatched its run (navigation only). */
+    parent?: {
+        conversationId: string;
+        runId: string;
+        callId: string;
+    };
+    /** Phase D: a fork child's case: the skill a person invoked that the child ran. */
+    invokedSkill?: {
+        id: string;
+        version: number;
+        name: string;
+    };
+}
+export interface SampleInput {
+    from?: string;
+    to?: string;
+    /** 1–50 */
+    count: number;
+    /** 缺省 `completed`；`any` = completed / failed / aborted。 */
+    status?: "completed" | "failed" | "any";
+    /** 该运行至少有多少次工具调用（按 `tool_result` 事件计）。 */
+    minToolCalls?: number;
+}
+/** 用例标题：触发消息文本的前 80 个字符（按码点）。 */
+export declare const CASE_TITLE_CHARS = 80;
+/**
+ * 快照运行的触发消息（该运行 `run_started` 之前最近的一条 `user_message`）的文本，空白折叠后取前 80 个字符。
+ */
+export declare function caseTitle(s: CaseSnapshot): string;
+/**
+ * 评测集与用例（D4、C3、C4）：组织管理员从某个 agent 的真实对话里挑选或抽样运行，每个用例存一份日志快照
+ * （`takeSnapshot`，≤ `evals.maxCaseBytes`），之后与原对话无关。写操作在事务里锁组织行并复核角色
+ * （`evals.manage`），再锁集合行，串行化同一集合的容量判定。查看用例内容写审计 `trajectory.viewed`。
+ */
+export declare class EvalSetsService {
+    private readonly db;
+    private readonly config;
+    private readonly prices;
+    private readonly settings;
+    constructor(db: KernelDb, config: () => KernelConfig, prices: PricesService, settings: SettingsService);
+    private views;
+    private static toView;
+    private view;
+    create(orgId: string, actor: string, input: {
+        agentId: string;
+        name: string;
+        description?: string;
+        rubric?: string;
+    }, actorTokenId?: string): Promise<EvalSetView>;
+    list(orgId: string): Promise<EvalSetView[]>;
+    get(orgId: string, setId: string): Promise<EvalSetDetail>;
+    update(orgId: string, actor: string, setId: string, patch: {
+        name?: string;
+        description?: string;
+        rubric?: string | null;
+    }, actorTokenId?: string): Promise<EvalSetView>;
+    /**
+     * 删除评测集（Ruling S1-a）：集合行软删除，留作审计存根；用例（含快照）硬删，结果随之级联删除，不留复制来的对话内容；
+     * 在执行的运行取消，运行行保留（无入口）。
+     */
+    remove(orgId: string, actor: string, setId: string, actorTokenId?: string): Promise<void>;
+    /** 未删除的评测集；不存在 → `eval.set_not_found`。 */
+    private liveSet;
+    private caseCount;
+    /**
+     * 在事务里写入一个用例：锁组织行并复核角色、锁集合行、按 `evals.maxCasesPerSet` 判容量，
+     * `(set_id, source_run_id)` 已存在时返回 null（不写）。成功时写审计 `eval.case_added`。
+     */
+    private insertCase;
+    /**
+     * 把某次运行加为用例。对话不在本组织（或是 eval 对话）→ `conversation.not_found`；对话的 agent 不是集合的 agent →
+     * `eval.agent_mismatch`；已在集合里 → `eval.case_exists`；集合满了 → `eval.too_many_cases`；
+     * 运行未终态 / 快照超大 → `eval.run_not_finished` / `eval.case_too_large`（`takeSnapshot`）。
+     */
+    addCase(orgId: string, actor: string, setId: string, input: {
+        conversationId: string;
+        runId: string;
+    }, actorTokenId?: string): Promise<EvalCaseView>;
+    /**
+     * 抽样：从该 agent 的 chat 对话里随机取（`ORDER BY random()`）已终态、尚不在集合里的运行，至多
+     * `min(count, 剩余容量)` 个，逐个取快照写入。快照超大（或其间被删除 / 写入时已满 / 并发加入）的计入 `skipped`。
+     * 集合已满 → `eval.too_many_cases`。逐个写入，内存里一次只有一份快照。
+     */
+    sample(orgId: string, actor: string, setId: string, input: SampleInput, actorTokenId?: string): Promise<{
+        added: number;
+        skipped: number;
+    }>;
+    removeCase(orgId: string, actor: string, setId: string, caseId: string, actorTokenId?: string): Promise<void>;
+    /**
+     * 用例详情：由快照还原轨迹（原对话删除后照常可读）。快照即对话内容，每次查看写审计 `trajectory.viewed`
+     * （target = 原对话 id，detail 带 `caseId` / `setId`）。
+     */
+    getCase(orgId: string, actor: string, setId: string, caseId: string, actorTokenId?: string): Promise<EvalCaseDetail>;
+}