@capybara-hq/kernel 0.1.0-rc.5 → 0.1.0-rc.7

package/dist/db/kernel-schema.d.ts

@@ -16,6 +16,25 @@ export interface AccountsTable {
     updated_at: Ts;
     /** 0064: the org the account last chose as current; a new session starts in it. */
     last_org_id: ColumnType<string | null, string | null | undefined, string | null>;
+    /** 0091: the person's time zone (IANA); null follows the browser (run context spec §3.4). */
+    time_zone: ColumnType<string | null, string | null | undefined, string | null>;
+    /** 0090: the account must set its own password first (a generated initial password). */
+    must_change_password: Generated<boolean>;
+}
+/**
+ * 0090: a generated initial password sealed for the person who approved the account's creation,
+ * read once on the web (Platform tool spec §14.5 item 2).
+ */
+export interface AccountInitialPasswordsTable {
+    account_id: string;
+    recipient_account_id: string;
+    conversation_id: string;
+    run_id: string;
+    call_id: string;
+    master_key_id: string;
+    ciphertext: Buffer;
+    expires_at: Date | string;
+    created_at: Ts;
 }
 export interface IdentitiesTable {
     source: string;
@@ -1012,8 +1031,31 @@ export interface UsageReportSweepTable {
     watermark: Ts;
     updated_at: ColumnType<Date, Date | string | undefined, Date | string>;
 }
+/** A model disabled on the platform (`org_id` null) or in one org (0089, spec B §5.3). */
+export interface ModelDisabledTable {
+    org_id: string | null;
+    provider: string;
+    model: string;
+    disabled_by: string | null;
+    disabled_at: Ts;
+}
+/** One model call outside a conversation, e.g. an extension's embed call (0089, spec B §5.4). */
+export interface ModelUsageTable {
+    id: Generated<string>;
+    org_id: string;
+    extension: string;
+    purpose: string;
+    provider: string;
+    model: string;
+    input_tokens: number;
+    occurred_at: Ts;
+    reported_at: TsNullable;
+    /** Set when the row's `usage.report` job gave up after its last attempt (the sweep skips it). */
+    report_failed_at: TsNullable;
+}
 export interface KernelDatabase {
     accounts: AccountsTable;
+    account_initial_passwords: AccountInitialPasswordsTable;
     identities: IdentitiesTable;
     local_passwords: LocalPasswordsTable;
     orgs: OrgsTable;
@@ -1071,6 +1113,8 @@ export interface KernelDatabase {
     eval_runs: EvalRunsTable;
     eval_results: EvalResultsTable;
     model_purpose_mappings: ModelPurposeMappingsTable;
+    model_disabled: ModelDisabledTable;
+    model_usage: ModelUsageTable;
     extension_packages: ExtensionPackagesTable;
     extension_blobs: ExtensionBlobsTable;
     extension_installs: ExtensionInstallsTable;

package/dist/environments/environments.schemas.d.ts

@@ -0,0 +1,3 @@
+import type { RouteSchemas } from "../platform/route-schemas.js";
+/** The environments routes the Platform operations mirror (`check:platform-ops` keys). */
+export declare const ENVIRONMENT_ROUTES: readonly RouteSchemas[];

package/dist/environments/environments.schemas.js

@@ -0,0 +1,14 @@
+import { EnvironmentNameBody, EnvironmentParams } from "./dto.js";
+/** The environments routes the Platform operations mirror (`check:platform-ops` keys). */
+export const ENVIRONMENT_ROUTES = [
+    { route: "GET /api/v1/orgs/:orgId/environments" },
+    { route: "POST /api/v1/orgs/:orgId/environments", body: EnvironmentNameBody },
+    { route: "GET /api/v1/orgs/:orgId/environments/:id", params: EnvironmentParams },
+    {
+        route: "PATCH /api/v1/orgs/:orgId/environments/:id",
+        params: EnvironmentParams,
+        body: EnvironmentNameBody,
+    },
+    { route: "DELETE /api/v1/orgs/:orgId/environments/:id", params: EnvironmentParams },
+    { route: "POST /api/v1/orgs/:orgId/environments/:id/files/reset", params: EnvironmentParams },
+];

package/dist/environments/environments.service.d.ts

@@ -72,10 +72,22 @@ export declare class EnvironmentsService {
      */
     create(orgId: string, accountId: string, input: {
         name: string;
-    }, actorTokenId?: string): Promise<EnvironmentView>;
+    }, actorTokenId?: string, auditDetail?: Record<string, unknown>): Promise<EnvironmentView>;
     private insert;
+    /**
+     * The refusals {@link create} would raise, read without locks or writes (a precheck before a
+     * confirmation card): no sandbox provider, the person's limit, a name already taken. Throws the
+     * same {@link ApiError}s; a race still ends in `create`'s own refusal.
+     */
+    checkCreate(orgId: string, accountId: string, name: string): Promise<void>;
+    /** How many of the person's conversations work in the environment (read only). */
+    boundConversations(orgId: string, accountId: string, id: string): Promise<number>;
+    /** The refusals {@link resetFiles} would raise (not found, its provider not current), read only. */
+    checkReset(orgId: string, accountId: string, id: string): Promise<void>;
+    /** The refusals {@link rename} would raise, read without locks (see {@link checkCreate}). */
+    checkRename(orgId: string, accountId: string, id: string, name: string): Promise<void>;
     /** Renames one of the person's live environments (the default too); see {@link create}. */
-    rename(orgId: string, accountId: string, id: string, name: string, actorTokenId?: string): Promise<EnvironmentView>;
+    rename(orgId: string, accountId: string, id: string, name: string, actorTokenId?: string, auditDetail?: Record<string, unknown>): Promise<EnvironmentView>;
     private renameIn;
     /**
      * Deletes one of the person's live environments: soft delete, its IM default row cleared, and
@@ -83,7 +95,7 @@ export declare class EnvironmentsService {
      * destroys the instance and the files). Conversations bound to it keep pointing at it; their
      * sandbox tools then fail with `environment.not_found`.
      */
-    remove(orgId: string, accountId: string, id: string, actorTokenId?: string): Promise<void>;
+    remove(orgId: string, accountId: string, id: string, actorTokenId?: string, auditDetail?: Record<string, unknown>): Promise<void>;
     /**
      * Starts one of the person's environments over with empty files (after
      * `environment.files_lost`): records the request (audit), then, after the commit, enqueues the
@@ -92,7 +104,7 @@ export declare class EnvironmentsService {
      * provider must be the current one (`environment.unavailable`). An enqueue failure is thrown
      * (the request can be repeated: the reset is idempotent).
      */
-    resetFiles(orgId: string, accountId: string, id: string, actorTokenId?: string): Promise<void>;
+    resetFiles(orgId: string, accountId: string, id: string, actorTokenId?: string, auditDetail?: Record<string, unknown>): Promise<void>;
     /** The check and audit row of {@link resetFiles}, committed before the job is enqueued. */
     private recordReset;
     /**

package/dist/environments/environments.service.js

@@ -94,11 +94,11 @@ let EnvironmentsService = class EnvironmentsService {
      * (case-insensitive) → `environment.name_taken`; `environments.maxPersonal` live ones (the
      * default counts) → `environment.limit_reached`. Serialized per person by their membership row.
      */
-    async create(orgId, accountId, input, actorTokenId) {
-        const id = await this.insert(orgId, accountId, input.name, actorTokenId);
+    async create(orgId, accountId, input, actorTokenId, auditDetail) {
+        const id = await this.insert(orgId, accountId, input.name, actorTokenId, auditDetail);
         return this.get(orgId, accountId, id);
     }
-    async insert(orgId, accountId, name, actorTokenId) {
+    async insert(orgId, accountId, name, actorTokenId, auditDetail) {
         const current = this.providers.current();
         if (!current)
             throw environmentUnavailable(null);
@@ -142,16 +142,60 @@ let EnvironmentsService = class EnvironmentsService {
             orgId,
             action: "environment.created",
             target: id,
-            detail: { name },
+            detail: { name, ...auditDetail },
         });
         return id;
     }
+    /**
+     * The refusals {@link create} would raise, read without locks or writes (a precheck before a
+     * confirmation card): no sandbox provider, the person's limit, a name already taken. Throws the
+     * same {@link ApiError}s; a race still ends in `create`'s own refusal.
+     */
+    async checkCreate(orgId, accountId, name) {
+        if (!this.providers.current())
+            throw environmentUnavailable(null);
+        const { n } = await this.db
+            .selectFrom("environments")
+            .select((eb) => eb.fn.countAll().as("n"))
+            .where("org_id", "=", orgId)
+            .where("owner_kind", "=", "personal")
+            .where("owner_account_id", "=", accountId)
+            .where("deleted_at", "is", null)
+            .executeTakeFirstOrThrow();
+        const max = this.config.get().environments.maxPersonal;
+        if (Number(n) >= max) {
+            throw new ApiError("environment.limit_reached", `You already have ${max} environments in this organization; delete one to create another`);
+        }
+        await this.assertNameFree(orgId, accountId, name);
+    }
+    /** How many of the person's conversations work in the environment (read only). */
+    async boundConversations(orgId, accountId, id) {
+        const { n } = await this.db
+            .selectFrom("conversations")
+            .select((eb) => eb.fn.countAll().as("n"))
+            .where("org_id", "=", orgId)
+            .where("account_id", "=", accountId)
+            .where("environment_id", "=", id)
+            .executeTakeFirstOrThrow();
+        return Number(n);
+    }
+    /** The refusals {@link resetFiles} would raise (not found, its provider not current), read only. */
+    async checkReset(orgId, accountId, id) {
+        await this.access.usable(orgId, accountId, id);
+    }
+    /** The refusals {@link rename} would raise, read without locks (see {@link checkCreate}). */
+    async checkRename(orgId, accountId, id, name) {
+        const before = await this.get(orgId, accountId, id);
+        if (before.name?.toLowerCase() !== name.toLowerCase()) {
+            await this.assertNameFree(orgId, accountId, name);
+        }
+    }
     /** Renames one of the person's live environments (the default too); see {@link create}. */
-    async rename(orgId, accountId, id, name, actorTokenId) {
-        await this.renameIn(orgId, accountId, id, name, actorTokenId);
+    async rename(orgId, accountId, id, name, actorTokenId, auditDetail) {
+        await this.renameIn(orgId, accountId, id, name, actorTokenId, auditDetail);
         return this.get(orgId, accountId, id);
     }
-    async renameIn(orgId, accountId, id, name, actorTokenId) {
+    async renameIn(orgId, accountId, id, name, actorTokenId, auditDetail) {
         const before = await this.lockOwn(orgId, accountId, id);
         if (before.name === name)
             return;
@@ -177,7 +221,7 @@ let EnvironmentsService = class EnvironmentsService {
             orgId,
             action: "environment.renamed",
             target: id,
-            detail: { from: before.name, to: name },
+            detail: { from: before.name, to: name, ...auditDetail },
         });
     }
     /**
@@ -186,7 +230,7 @@ let EnvironmentsService = class EnvironmentsService {
      * destroys the instance and the files). Conversations bound to it keep pointing at it; their
      * sandbox tools then fail with `environment.not_found`.
      */
-    async remove(orgId, accountId, id, actorTokenId) {
+    async remove(orgId, accountId, id, actorTokenId, auditDetail) {
         const before = await this.lockOwn(orgId, accountId, id);
         await this.db
             .updateTable("environments")
@@ -205,7 +249,7 @@ let EnvironmentsService = class EnvironmentsService {
             orgId,
             action: "environment.deleted",
             target: id,
-            detail: { name: before.name, isDefault: before.is_default },
+            detail: { name: before.name, isDefault: before.is_default, ...auditDetail },
         });
         afterCommit(async () => {
             try {
@@ -225,12 +269,12 @@ let EnvironmentsService = class EnvironmentsService {
      * provider must be the current one (`environment.unavailable`). An enqueue failure is thrown
      * (the request can be repeated: the reset is idempotent).
      */
-    async resetFiles(orgId, accountId, id, actorTokenId) {
-        await this.recordReset(orgId, accountId, id, actorTokenId);
+    async resetFiles(orgId, accountId, id, actorTokenId, auditDetail) {
+        await this.recordReset(orgId, accountId, id, actorTokenId, auditDetail);
         await this.sandboxJobs.resetFiles(id);
     }
     /** The check and audit row of {@link resetFiles}, committed before the job is enqueued. */
-    async recordReset(orgId, accountId, id, actorTokenId) {
+    async recordReset(orgId, accountId, id, actorTokenId, auditDetail) {
         await this.access.usable(orgId, accountId, id);
         await audit(this.db.db, {
             actorAccountId: accountId,
@@ -238,6 +282,7 @@ let EnvironmentsService = class EnvironmentsService {
             orgId,
             action: "environment.files_reset",
             target: id,
+            ...(auditDetail !== undefined && { detail: auditDetail }),
         });
     }
     /**

package/dist/errors.d.ts

@@ -15,6 +15,8 @@ export declare const ERROR_CODES: {
     readonly "account.login_taken": 409;
     readonly "account.not_found": 404;
     readonly "account.last_super_admin": 409;
+    readonly "account.initial_password_unavailable": 404;
+    readonly "account.create_needs_approval": 409;
     readonly "auth.account_disabled": 403;
     readonly "auth.required": 401;
     readonly "auth.token_expired": 401;
@@ -25,6 +27,7 @@ export declare const ERROR_CODES: {
     readonly "auth.csrf": 403;
     readonly "auth.rate_limited": 429;
     readonly "auth.session_required": 403;
+    readonly "auth.password_change_required": 403;
     readonly "token.not_found": 404;
     readonly "org.not_found": 404;
     readonly "org.creation_disabled": 403;
@@ -280,6 +283,9 @@ export declare const ERROR_CODES: {
     readonly "eval.candidate_invalid": 400;
     readonly "model.unconfigured": 409;
     readonly "model.purpose_unknown": 404;
+    readonly "model.provider_not_found": 404;
+    /** An extension's `ctx.models.embed` call was denied by a `model.request` policy hook. */
+    readonly "model.hook_denied": 403;
     readonly "eval.result_not_found": 404;
 };
 export type ErrorCode = keyof typeof ERROR_CODES;

package/dist/errors.js

@@ -15,6 +15,8 @@ export const ERROR_CODES = {
     "account.login_taken": 409,
     "account.not_found": 404,
     "account.last_super_admin": 409,
+    "account.initial_password_unavailable": 404,
+    "account.create_needs_approval": 409,
     "auth.account_disabled": 403,
     "auth.required": 401,
     "auth.token_expired": 401,
@@ -25,6 +27,7 @@ export const ERROR_CODES = {
     "auth.csrf": 403,
     "auth.rate_limited": 429,
     "auth.session_required": 403,
+    "auth.password_change_required": 403,
     "token.not_found": 404,
     "org.not_found": 404,
     "org.creation_disabled": 403,
@@ -280,6 +283,9 @@ export const ERROR_CODES = {
     "eval.candidate_invalid": 400,
     "model.unconfigured": 409,
     "model.purpose_unknown": 404,
+    "model.provider_not_found": 404,
+    /** An extension's `ctx.models.embed` call was denied by a `model.request` policy hook. */
+    "model.hook_denied": 403,
     "eval.result_not_found": 404,
 };
 /** 业务代码抛出的 API 错误；由 HTTP 层的错误过滤器转成错误信封。 */

package/dist/evals/evals.schemas.d.ts

@@ -0,0 +1,110 @@
+import { z } from "zod";
+import type { RouteSchemas } from "../platform/route-schemas.js";
+export declare const EvalSetCreateBody: z.ZodObject<{
+    agentId: z.ZodString;
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    rubric: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const EvalSetPatchBody: z.ZodObject<{
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    rubric: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+}, z.core.$strict>;
+export declare const EvalAddCaseBody: z.ZodObject<{
+    conversationId: z.ZodString;
+    runId: z.ZodString;
+}, z.core.$strict>;
+export declare const EvalSampleBody: z.ZodObject<{
+    from: z.ZodOptional<z.ZodISODateTime>;
+    to: z.ZodOptional<z.ZodISODateTime>;
+    count: z.ZodNumber;
+    status: z.ZodOptional<z.ZodEnum<{
+        any: "any";
+        completed: "completed";
+        failed: "failed";
+    }>>;
+    minToolCalls: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const EvalCandidateSchema: z.ZodObject<{
+    agentVersion: z.ZodOptional<z.ZodNumber>;
+    model: z.ZodOptional<z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+    }, z.core.$strict>>;
+    modelParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+    instructions: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+/** `tokenCap` is only type-checked here; the service judges the range (`eval.token_cap_invalid`). */
+export declare const EvalRunEstimateBody: z.ZodObject<{
+    mode: z.ZodEnum<{
+        run: "run";
+        step: "step";
+    }>;
+    candidate: z.ZodObject<{
+        agentVersion: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strict>>;
+        modelParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+        instructions: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    judge: z.ZodDefault<z.ZodBoolean>;
+    tokenCap: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const EvalRunStartBody: z.ZodObject<{
+    mode: z.ZodEnum<{
+        run: "run";
+        step: "step";
+    }>;
+    candidate: z.ZodObject<{
+        agentVersion: z.ZodOptional<z.ZodNumber>;
+        model: z.ZodOptional<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strict>>;
+        modelParams: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+        instructions: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    judge: z.ZodDefault<z.ZodBoolean>;
+    tokenCap: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+/**
+ * `view=light`: no content blocks (result tables and polling; the response carries `asOf`);
+ * default `full`. `updatedSince` (ISO time) takes only what changed after it.
+ */
+export declare const EvalRunResultsQuery: z.ZodObject<{
+    after: z.ZodOptional<z.ZodString>;
+    limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    view: z.ZodDefault<z.ZodEnum<{
+        full: "full";
+        light: "light";
+    }>>;
+    updatedSince: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const EvalLabelBody: z.ZodObject<{
+    label: z.ZodNullable<z.ZodEnum<{
+        fail: "fail";
+        pass: "pass";
+    }>>;
+    note: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const EvalSetParams: z.ZodObject<{
+    setId: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalCaseParams: z.ZodObject<{
+    setId: z.ZodString;
+    caseId: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalRunParams: z.ZodObject<{
+    runId: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalResultParams: z.ZodObject<{
+    runId: z.ZodString;
+    resultId: z.ZodString;
+}, z.core.$strip>;
+export declare const EvalLabelParams: z.ZodObject<{
+    resultId: z.ZodString;
+}, z.core.$strip>;
+export declare const EVAL_ROUTES: readonly RouteSchemas[];

package/dist/evals/evals.schemas.js

@@ -0,0 +1,107 @@
+import { z } from "zod";
+import { ModelRef } from "../agents/definition.js";
+const NAME = z.string().trim().min(1).max(200);
+const DESCRIPTION = z.string().max(2000);
+const RUBRIC = z.string().min(1).max(20_000);
+export const EvalSetCreateBody = z
+    .object({
+    agentId: z.string().min(1).max(100),
+    name: NAME,
+    description: DESCRIPTION.optional(),
+    rubric: RUBRIC.optional(),
+})
+    .strict();
+export const EvalSetPatchBody = z
+    .object({
+    name: NAME.optional(),
+    description: DESCRIPTION.optional(),
+    rubric: RUBRIC.nullable().optional(),
+})
+    .strict();
+export const EvalAddCaseBody = z
+    .object({ conversationId: z.string().min(1).max(100), runId: z.string().min(1).max(100) })
+    .strict();
+export const EvalSampleBody = z
+    .object({
+    from: z.iso.datetime({ offset: true }).optional(),
+    to: z.iso.datetime({ offset: true }).optional(),
+    count: z.number().int().min(1).max(50),
+    status: z.enum(["completed", "failed", "any"]).optional(),
+    minToolCalls: z.number().int().min(0).max(1000).optional(),
+})
+    .strict();
+export const EvalCandidateSchema = z
+    .object({
+    agentVersion: z.number().int().min(1).optional(),
+    model: ModelRef.strict().optional(),
+    modelParams: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+    instructions: z.string().max(100_000).optional(),
+})
+    .strict();
+/** `tokenCap` is only type-checked here; the service judges the range (`eval.token_cap_invalid`). */
+export const EvalRunEstimateBody = z
+    .object({
+    mode: z.enum(["step", "run"]),
+    candidate: EvalCandidateSchema,
+    judge: z.boolean().default(false),
+    tokenCap: z.number().int().optional(),
+})
+    .strict();
+export const EvalRunStartBody = EvalRunEstimateBody;
+/**
+ * `view=light`: no content blocks (result tables and polling; the response carries `asOf`);
+ * default `full`. `updatedSince` (ISO time) takes only what changed after it.
+ */
+export const EvalRunResultsQuery = z
+    .object({
+    after: z.string().max(100).optional(),
+    limit: z.coerce.number().int().min(1).max(200).default(50),
+    view: z.enum(["full", "light"]).default("full"),
+    updatedSince: z
+        .string()
+        .max(64)
+        .refine((v) => !Number.isNaN(Date.parse(v)), { message: "must be an ISO timestamp" })
+        .optional(),
+})
+    .strict();
+export const EvalLabelBody = z
+    .object({
+    label: z.enum(["pass", "fail"]).nullable(),
+    note: z.string().max(2000).optional(),
+})
+    .strict();
+/** The path params the eval routes read (`@Param`), bounded as the operations take them. */
+const Id = z.string().min(1).max(100);
+export const EvalSetParams = z.object({ setId: Id });
+export const EvalCaseParams = z.object({ setId: Id, caseId: Id });
+export const EvalRunParams = z.object({ runId: Id });
+export const EvalResultParams = z.object({ runId: Id, resultId: Id });
+export const EvalLabelParams = z.object({ resultId: Id });
+const O = "/api/v1/orgs/:orgId";
+export const EVAL_ROUTES = [
+    { route: `POST ${O}/eval-sets`, body: EvalSetCreateBody },
+    { route: `GET ${O}/eval-sets` },
+    { route: `GET ${O}/eval-sets/:setId`, params: EvalSetParams },
+    { route: `PATCH ${O}/eval-sets/:setId`, params: EvalSetParams, body: EvalSetPatchBody },
+    { route: `DELETE ${O}/eval-sets/:setId`, params: EvalSetParams },
+    { route: `POST ${O}/eval-sets/:setId/cases`, params: EvalSetParams, body: EvalAddCaseBody },
+    {
+        route: `POST ${O}/eval-sets/:setId/cases/sample`,
+        params: EvalSetParams,
+        body: EvalSampleBody,
+    },
+    { route: `GET ${O}/eval-sets/:setId/cases/:caseId`, params: EvalCaseParams },
+    { route: `DELETE ${O}/eval-sets/:setId/cases/:caseId`, params: EvalCaseParams },
+    {
+        route: `POST ${O}/eval-sets/:setId/runs/estimate`,
+        params: EvalSetParams,
+        body: EvalRunEstimateBody,
+    },
+    { route: `POST ${O}/eval-sets/:setId/runs`, params: EvalSetParams, body: EvalRunStartBody },
+    { route: `GET ${O}/eval-sets/:setId/runs`, params: EvalSetParams },
+    { route: `GET ${O}/eval-runs/:runId`, params: EvalRunParams },
+    { route: `GET ${O}/eval-runs/:runId/results`, params: EvalRunParams, query: EvalRunResultsQuery },
+    { route: `GET ${O}/eval-runs/:runId/results/:resultId`, params: EvalResultParams },
+    { route: `POST ${O}/eval-runs/:runId/cancel`, params: EvalRunParams },
+    { route: `PUT ${O}/eval-results/:resultId/label`, params: EvalLabelParams, body: EvalLabelBody },
+];

package/dist/evals/reconstruct.js

@@ -1,5 +1,5 @@
 import { sanitizeParams, } from "@capybara-hq/plugin-sdk";
-import { AgentDefinitionSchema, normalizeStoredDefinition, } from "../agents/definition.js";
+import { AgentDefinitionSchema, builtinDisabled, canonicalDisabledTools, normalizeStoredDefinition, } from "../agents/definition.js";
 import { sameModel, selectionAt } from "../conversations/selection.js";
 import { ApiError } from "../errors.js";
 import { mediaRefs, planMedia, } from "../runtime/context/build.js";
@@ -217,18 +217,25 @@ const sameTemplate = (logged, t) => logged.source !== "plugin" ||
         (logged.platformTemplateId ?? null) === (t.platformTemplateId ?? null));
 const pinsOf = (d) => [...new Set(d.skills?.pinned ?? [])].slice(0, PINNED_SKILLS_LIMITS.count);
 /**
- * Tool refs as a sub-agent gets them without template narrowing (general and org agents).
- * `Agent` depends on the child's depth, not on the definition (agent map spec §4.1): it is kept
- * only when the logged child had it, so a child logged without it (at the depth limit, or before
- * nesting) compares equal to a candidate that offers it.
+ * A sub-agent's tools without template narrowing (general and org agents), as compared: its
+ * turned-off tools (tools default-on spec §4) and its key filter's allowlist and `deny` keys,
+ * {@link CHILD_REMOVED_TOOLS} left out. `Agent` depends on the child's depth, not on the
+ * definition (agent map spec §4.1): a `deny` of it never counts, and a turned-off `Agent` counts
+ * only when the logged child had it (`keepAgent`), so a child logged without it (at the depth
+ * limit, or before nesting) compares equal to a candidate that offers it.
  */
-const childToolsOf = (d, keepAgent) => canonicalJson(d.tools.filter((t) => t.kind !== "builtin" ||
-    (!CHILD_REMOVED_TOOLS.includes(t.name) &&
-        (keepAgent || t.name !== AGENT_TOOL))));
-const hasAgent = (d) => d.tools.some((t) => t.kind === "builtin" && t.name === AGENT_TOOL);
+const childToolsOf = (d, keepAgent) => canonicalJson({
+    disabled: canonicalDisabledTools((d.disabledTools ?? []).filter((r) => keepAgent || r.kind !== "builtin" || r.name !== AGENT_TOOL)),
+    allow: d.subagentTools?.allow ?? null,
+    deny: (d.subagentTools?.deny ?? []).filter((k) => !CHILD_REMOVED_TOOLS.includes(k) && k !== AGENT_TOOL),
+});
+/** Does the child have `Agent` by its definition (not turned off, not removed by its filter)? */
+const hasAgent = (d) => !builtinDisabled(d.disabledTools, AGENT_TOOL) &&
+    !(d.subagentTools?.deny ?? []).includes(AGENT_TOOL) &&
+    (d.subagentTools?.allow === undefined || d.subagentTools.allow.includes(AGENT_TOOL));
 /** The parent's selection a template child depends on: its tools, org skills and roster entry. */
 const parentSide = (d, started) => canonicalJson({
-    tools: d.tools,
+    disabledTools: canonicalDisabledTools(d.disabledTools ?? []),
     orgSkills: d.skills?.orgSkills ?? [],
     listed: started.template.source === "general"
         ? d.subagents?.general !== false
@@ -264,7 +271,7 @@ original) {
         (pinned !== null && canonicalJson(pinned) !== canonicalJson(pinsOf(logged))) ||
         (tools !== null && tools !== childTools(logged));
     if (sameVersion) {
-        return differs(candidate.instructions, pinsOf(candidate), canonicalJson(candidate.tools));
+        return differs(candidate.instructions, pinsOf(candidate), childTools(candidate));
     }
     switch (started.template.source) {
         case "agent":

package/dist/evals/recorded-tools.d.ts

@@ -31,7 +31,8 @@ export interface RecordedCall {
  * - a connector: the tool definition the key was recorded with (`toolSpecs`, found by the specHash of
  *   its `tool_call_started` or its `model_call` tools / catalog entry) has `connectorData`;
  * - an extension: that definition exists without `connectorData` and its name is the key's tool
- *   part, or an agent definition names exactly that tool as a `{ kind: "plugin", plugin: x, name }` ref.
+ *   part, or an rc-era agent definition names exactly that tool as a `{ kind: "plugin", plugin: x, name }`
+ *   ref.
  */
 export declare function legacyScopeOf(s: CaseSnapshot): LegacyScope;
 /**

package/dist/evals/recorded-tools.js

@@ -16,7 +16,8 @@ export const REPLAY_NO_RECORDING = "[replay] No recorded result matches this too
  * - a connector: the tool definition the key was recorded with (`toolSpecs`, found by the specHash of
  *   its `tool_call_started` or its `model_call` tools / catalog entry) has `connectorData`;
  * - an extension: that definition exists without `connectorData` and its name is the key's tool
- *   part, or an agent definition names exactly that tool as a `{ kind: "plugin", plugin: x, name }` ref.
+ *   part, or an rc-era agent definition names exactly that tool as a `{ kind: "plugin", plugin: x, name }`
+ *   ref.
  */
 export function legacyScopeOf(s) {
     const hashes = new Map();
@@ -42,10 +43,17 @@ export function legacyScopeOf(s) {
     const extensionNames = new Set();
     const connectorSlugs = new Set();
     const planted = new Set();
+    // rc-era snapshots hold definitions from before tools were on by default: an enabled `tools`
+    // list of refs (tools default-on spec §2)
     for (const def of Object.values(s.definitions)) {
-        for (const t of def.tools)
-            if (t.kind === "plugin")
+        const refs = def.tools;
+        if (!Array.isArray(refs))
+            continue;
+        for (const t of refs) {
+            if (t?.kind === "plugin" && typeof t.plugin === "string" && typeof t.name === "string") {
                 planted.add(`${t.plugin}__${t.name}`);
+            }
+        }
     }
     for (const [key, set] of hashes) {
         const i = key.indexOf("__");

package/dist/evals/run-replay.d.ts

@@ -18,7 +18,7 @@ import type { ResolvedCandidate } from "./step-replay.js";
  * {@link RecordedToolPipeline}（只回放记录的结果，不执行任何工具、沙箱、连接器或审批）。
  *
  * - 种子：原运行 `run_started` 之前的事件按原 seq 插入（`last_seq` 停在其前一位，新的 `run_started` 因此与原来同 seq），
- *   原运行开始时的 `skills_catalog` 照抄；然后写 `runs` 行与 `run_started`，`agent` 用候选定义。
+ *   原运行开始时的 `skills_catalog` 与 `run_context` 照抄；然后写 `runs` 行与 `run_started`，`agent` 用候选定义。
  * - 插话：原运行中途吸收的可吸收事件（user_message, and finished background tasks: background Bash
  *   spec §4.4），在「原吸收步骤 ≤ 当前步骤」时追加进评测对话并交给 loop（a recorded finish is
  *   appended as recorded: no task starts, polls or stops; likewise a sub-agent's delivery, agent map
@@ -114,6 +114,13 @@ export interface ReplaySeed {
     catalog: {
         payload: unknown;
     } | undefined;
+    /**
+     * The original run's `run_context` event (run context spec §8), copied as is: a candidate sees
+     * the original date, names and org, never the eval day's.
+     */
+    runContext: {
+        payload: unknown;
+    } | undefined;
 }
 /**
  * The seed of a full-run replay (C6): the `kind = 'eval'` conversation with the events before the