@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/sandbox/lifecycle.d.ts

@@ -0,0 +1,21 @@
+import type { Logger } from "@capybara-hq/plugin-sdk";
+import type { Maintenance } from "../runtime/maintenance.js";
+import type { SandboxService } from "./sandbox.service.js";
+/** 空闲休眠扫描：任务名、job scheduler id（不含 `:`）与周期。 */
+export declare const SANDBOX_SUSPEND_JOB = "sandbox-suspend-idle";
+export declare const SANDBOX_SUSPEND_SCHEDULER_ID = "sandbox-suspend-idle-every-minute";
+export declare const SANDBOX_SUSPEND_EVERY_MS = 60000;
+/** 不活跃销毁扫描：任务名、job scheduler id（不含 `:`）与周期。 */
+export declare const SANDBOX_DESTROY_JOB = "sandbox-destroy-inactive";
+export declare const SANDBOX_DESTROY_SCHEDULER_ID = "sandbox-destroy-inactive-hourly";
+export declare const SANDBOX_DESTROY_EVERY_MS = 3600000;
+/** 一次性任务：立即销毁一批对话的沙箱（成员离职后投递，数据 `{ conversationIds }`）。 */
+export declare const SANDBOX_DESTROY_FOR_JOB = "sandbox-destroy-for";
+/** 投递 {@link SANDBOX_DESTROY_FOR_JOB}：失败的整批重试（已销毁的再调一次什么也不做）。 */
+export declare function scheduleSandboxDestroy(maintenance: Pick<Maintenance, "add">, conversationIds: string[]): Promise<void>;
+/**
+ * 在维护队列（`cap-kernel-maintenance`）登记沙箱的两个周期任务（worker 角色执行）：
+ * 每分钟休眠空闲 / 连续运行超时的沙箱，每小时销毁不活跃、组织已删除或对话已删除的沙箱。
+ * 没有沙箱提供方时两个任务什么也不做。另登记一次性任务 {@link SANDBOX_DESTROY_FOR_JOB}（成员离职）。
+ */
+export declare function registerSandboxMaintenance(maintenance: Maintenance, sandbox: SandboxService, log: Pick<Logger, "info" | "warn">): void;

package/dist/sandbox/lifecycle.js

@@ -0,0 +1,68 @@
+/** 空闲休眠扫描：任务名、job scheduler id（不含 `:`）与周期。 */
+export const SANDBOX_SUSPEND_JOB = "sandbox-suspend-idle";
+export const SANDBOX_SUSPEND_SCHEDULER_ID = "sandbox-suspend-idle-every-minute";
+export const SANDBOX_SUSPEND_EVERY_MS = 60_000;
+/** 不活跃销毁扫描：任务名、job scheduler id（不含 `:`）与周期。 */
+export const SANDBOX_DESTROY_JOB = "sandbox-destroy-inactive";
+export const SANDBOX_DESTROY_SCHEDULER_ID = "sandbox-destroy-inactive-hourly";
+export const SANDBOX_DESTROY_EVERY_MS = 3_600_000;
+/** 一次性任务：立即销毁一批对话的沙箱（成员离职后投递，数据 `{ conversationIds }`）。 */
+export const SANDBOX_DESTROY_FOR_JOB = "sandbox-destroy-for";
+/** 投递 {@link SANDBOX_DESTROY_FOR_JOB}：失败的整批重试（已销毁的再调一次什么也不做）。 */
+export async function scheduleSandboxDestroy(maintenance, conversationIds) {
+    await maintenance.add(SANDBOX_DESTROY_FOR_JOB, { conversationIds }, {
+        attempts: 5,
+        backoff: { type: "exponential", delay: 30_000 },
+        removeOnComplete: true,
+        removeOnFail: 100,
+    });
+}
+/**
+ * 在维护队列（`cap-kernel-maintenance`）登记沙箱的两个周期任务（worker 角色执行）：
+ * 每分钟休眠空闲 / 连续运行超时的沙箱，每小时销毁不活跃、组织已删除或对话已删除的沙箱。
+ * 没有沙箱提供方时两个任务什么也不做。另登记一次性任务 {@link SANDBOX_DESTROY_FOR_JOB}（成员离职）。
+ */
+export function registerSandboxMaintenance(maintenance, sandbox, log) {
+    maintenance.handle(SANDBOX_DESTROY_FOR_JOB, async (job) => {
+        const ids = job.data.conversationIds;
+        if (!Array.isArray(ids))
+            return;
+        let failed = 0;
+        for (const id of ids) {
+            if (typeof id !== "string")
+                continue;
+            try {
+                await sandbox.destroyFor(id);
+            }
+            catch (err) {
+                failed++;
+                log.warn("cannot destroy sandbox of an offboarded member; will retry", {
+                    conversationId: id,
+                    err,
+                });
+            }
+        }
+        if (failed > 0)
+            throw new Error(`${failed} of ${ids.length} sandboxes could not be destroyed`);
+    });
+    maintenance.handle(SANDBOX_SUSPEND_JOB, async () => {
+        const n = await sandbox.suspendIdle(new Date());
+        if (n > 0)
+            log.info("idle sandboxes suspended", { count: n });
+    });
+    maintenance.schedule({
+        id: SANDBOX_SUSPEND_SCHEDULER_ID,
+        name: SANDBOX_SUSPEND_JOB,
+        everyMs: () => SANDBOX_SUSPEND_EVERY_MS,
+    });
+    maintenance.handle(SANDBOX_DESTROY_JOB, async () => {
+        const n = await sandbox.destroyInactive(new Date());
+        if (n > 0)
+            log.info("inactive sandboxes destroyed", { count: n });
+    });
+    maintenance.schedule({
+        id: SANDBOX_DESTROY_SCHEDULER_ID,
+        name: SANDBOX_DESTROY_JOB,
+        everyMs: () => SANDBOX_DESTROY_EVERY_MS,
+    });
+}

package/dist/sandbox/sandbox.service.d.ts

@@ -0,0 +1,249 @@
+import { type Logger, type SandboxHandle, type SandboxProvider } from "@capybara-hq/plugin-sdk";
+import type { Redis } from "ioredis";
+import { type Selectable } from "kysely";
+import { type KernelConfig } from "../config/schema.js";
+import type { KernelDb, SandboxesTable } from "../db/kernel-schema.js";
+import { ApiError } from "../errors.js";
+import { SandboxLeases, type SandboxPathLease } from "./leases.js";
+export { SANDBOX_BUSY_PREFIX, SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
+/** `withSandbox` 回调的第三个参数。 */
+export interface SandboxUseInfo {
+    /** 本次调用新建了沙箱（首次使用或重建）。 */
+    fresh: boolean;
+    /** 之前有过实例、但已不在（被销毁、提供方返回 gone、换了提供方），这是重建的空沙箱。 */
+    recreated: boolean;
+}
+/** 当前的 `sandbox@1` 提供方：插件名与实现。 */
+export interface SandboxProviderEntry {
+    plugin: string;
+    impl: SandboxProvider;
+}
+export interface SandboxServiceDeps {
+    db: KernelDb;
+    redis: Redis;
+    /** 每次调用时取（提供方在插件加载后才就绪）；没有提供方为 undefined。 */
+    provider: () => SandboxProviderEntry | undefined;
+    /** 按插件名找仍加载着的提供方（销毁换提供方前的旧实例）；省略则视为找不到。 */
+    lookup?: (plugin: string) => SandboxProvider | undefined;
+    config: () => Pick<KernelConfig, "sandbox">;
+    log: Pick<Logger, "info" | "warn">;
+    /** Poll interval while waiting for a lease (ms), default 100; tests make it shorter. */
+    lockPollMs?: number;
+    /** Lease TTL (ms), default {@link SANDBOX_LOCK_TTL_MS}; tests make it shorter to notice a lost lease quickly. */
+    lockTtlMs?: number;
+    /** Lease implementation, default a {@link SandboxLeases} on `redis` (tests may wrap one). */
+    leases?: SandboxLeases;
+}
+type Row = Selectable<SandboxesTable>;
+/**
+ * One instance of a conversation's sandbox: its handle and the row's `created_at`, which the
+ * fenced write after every `create` sets. The handle alone is not enough: sandbox-docker's handle
+ * is the instance name, the same after a rebuild (spec §10.10).
+ */
+interface Instance {
+    handle: SandboxHandle;
+    createdAt: number;
+}
+/**
+ * The row still records the instance `gone` (same handle and same `created_at`). Handles compare
+ * canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
+ */
+export declare const isInstance: (row: Pick<Row, "handle" | "created_at">, gone: Instance | null) => boolean;
+/**
+ * Concurrent commands a provider gets (spec §10.7): `max` when it declares both `concurrentExec`
+ * and `killTagged`, else 1 (one call at a time, the behavior before phase C).
+ */
+export declare function sandboxConcurrency(impl: Pick<SandboxProvider, "concurrentExec" | "killTagged">, max: number): number;
+/**
+ * 启动时要提醒运维的沙箱配置问题（英文日志）：
+ * - `runtime.maxRunSeconds` 小于 `sandbox.limits.maxCommandSeconds`：长命令会被运行时限截断（Ruling M20）；
+ * - `sandbox.limits.egress` 为 open、但当前提供方插件没打开 `allowOpenEgress`：每次创建沙箱都会被拒。
+ * 两个时限都是默认值（900 < 7200，已知的取舍）时只记 info；运维改过其中一个却仍不一致时才 warn。
+ * - the provider declares only one of `concurrentExec` and `killTagged`: it is used one call at a
+ *   time (spec §10.7).
+ */
+export declare function sandboxConfigWarnings(cfg: Pick<KernelConfig, "runtime" | "sandbox" | "extensionConfig">, providerPlugin?: string, impl?: Pick<SandboxProvider, "concurrentExec" | "killTagged">): {
+    level: "info" | "warn";
+    message: string;
+}[];
+/** 没有沙箱提供方时的错误（503）。 */
+export declare const sandboxUnavailable: () => ApiError;
+/**
+ * 内核的沙箱服务（M5 T7）：在 `sandbox@1` 提供方之上，每个对话一个沙箱。
+ *
+ * - **懒创建**：第一次 {@link withSandbox} 才创建；状态在 PG 表 `sandboxes`。
+ * - **Leases** (phase C, spec §10, {@link SandboxLeases}): using a sandbox recorded `running` with
+ *   the current provider takes a **shared** lease, so up to `sandbox.maxConcurrentCommands` holders
+ *   (commands of parallel sub-agents) run at once ({@link sandboxConcurrency}: 1 for a provider
+ *   without `concurrentExec` + `killTagged`). Every lifecycle change (create, resume, rebuild,
+ *   suspend, destroy) takes the **exclusive** lease, alone; a caller that needs one on the use path
+ *   sets the intent so new shared leases wait for it, and downgrades to a shared lease to run its
+ *   own command once the sandbox is running. Waiting is capped at `sandbox.lockWaitSeconds`
+ *   (`sandbox.busy`); two replicas using it for the first time at once still build only one.
+ * - **防护**：exclusive holders read `generation` and compare-and-increment it with every write of
+ *   status / handle ({@link SandboxLockLostError}); every acquisition (shared ones too) bumps it
+ *   first, so a stale exclusive holder's later writes fail. Shared holders write nothing else but
+ *   `last_used_at` (monotonic). A lost
+ *   lease aborts the signal passed to `fn` and to maintenance provider calls.
+ * - **休眠与销毁**由维护队列定期调用 {@link suspendIdle} / {@link destroyInactive}；它们只试一次 exclusive
+ *   lease and skip a sandbox with any holder; a sandbox past `maxRunningMinutes` gets the intent for
+ *   a short window (about a second) so the suspension can get in when its commands end.
+ * - **透明恢复**：已休眠的在下次使用时恢复；恢复返回 `gone`、已销毁、记录的提供方不是当前提供方、
+ *   或 `fn` 抛 `SandboxGoneError`（重试一次）时重建空沙箱，`recreated` 为 true（工具层告诉模型「之前的文件已不在」）。
+ *   Several holders hitting `gone` together rebuild once: the others find the new instance.
+ * - **Takeover** (spec §10.5): every holder records itself in `capk:sandbox-holders:<conv>` (again
+ *   before each `exec`) and its commands carry its lease token as `tag`. An acquirer that finds a
+ *   record without a live lease (a crashed holder) kills its commands: `killTagged` on the shared
+ *   path whenever the provider has it (else `killAll` only while it is the one shared holder),
+ *   `killAll` under the exclusive lease (no live sibling can exist). The single marker
+ *   `capk:sandbox-busy:<conv>` of kernels before phase C sends the next use down the exclusive
+ *   path, which `killAll`s; holders of this kernel write it as `v2` for older replicas (§10.8),
+ *   which this kernel ignores. A sandbox just resumed from `suspended` is `killAll`ed too. Failures
+ *   are only warned (the in-sandbox time limit is the backstop).
+ * - **Instance identity**: a holder that hits `gone` names the instance by its handle and the
+ *   row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
+ *   after a rebuild on providers whose handle is the instance name (sandbox-docker).
+ *
+ * 换提供方会泄漏旧实例：新提供方销毁不了旧提供方的实例，只有旧提供方仍加载时才能清理（记在 `prev_*` 列）。
+ * 没有提供方时 {@link available} 为 false，使用时抛 `sandbox.unavailable`（503）。本身无进程内状态，可多副本共用。
+ */
+export declare class SandboxService {
+    private readonly d;
+    private readonly leases;
+    private readonly pollMs;
+    constructor(d: SandboxServiceDeps);
+    /** 是否有沙箱提供方。 */
+    get available(): boolean;
+    /** Concurrent holders the provider gets ({@link sandboxConcurrency} with the current config). */
+    effectiveCap(p: SandboxProviderEntry): number;
+    /**
+     * 该对话沙箱的对外状态（只读，不拿锁）：`running` / `suspended` 与最近使用时间。
+     * 还没有、已销毁、或停在 `creating`（正在创建，或创建失败等下次使用重建）的都返回 null（界面显示「未创建」）。
+     */
+    status(orgId: string, conversationId: string): Promise<{
+        status: "running" | "suspended";
+        lastUsedAt: Date;
+    } | null>;
+    /**
+     * A short lease on one path of the conversation's sandbox around the kernel's own
+     * read-modify-write of that file (spec §10.4); waits up to 30 s, then throws
+     * `SandboxPathBusyError`. Its `signal` aborts when a renewal finds it lost.
+     */
+    pathLease(conversationId: string, path: string, signal: AbortSignal): Promise<SandboxPathLease>;
+    /**
+     * 在该对话（the sandbox conversation）的沙箱上执行 `fn`: take a lease → make sure the sandbox runs
+     * (create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease → update
+     * `last_used_at` → release.
+     * - 等待超过 `sandbox.lockWaitSeconds` 抛 `sandbox.busy`（409）；`signal` 中止等待与创建 / 恢复。
+     * - `fn` 的第四个参数是 `signal` 与「lease 已丢失」的合并：执行命令时用它，lease 丢失就中止。
+     * - The provider passed to `fn` labels every `exec` with this holder's tag.
+     * - `fn` 抛 `SandboxGoneError`（实例已不在）时重建（or take the instance another holder rebuilt）、
+     *   以 `recreated: true` 再调用一次；再次 gone 则抛出。其余错误原样抛出。
+     * - `holderConversationId` (default `conversationId`): the agent conversation using it, a
+     *   sub-agent's own for a child. When that holder last used another instance (another holder
+     *   rebuilt it meanwhile), `info.recreated` is true, so it learns its files are gone (final
+     *   review m5).
+     */
+    withSandbox<T>(ctx: {
+        orgId: string;
+        conversationId: string;
+        holderConversationId?: string;
+    }, fn: (h: SandboxHandle, p: SandboxProvider, info: SandboxUseInfo, signal: AbortSignal) => Promise<T>, signal: AbortSignal): Promise<T>;
+    /**
+     * Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<conv>:<holder>`, its
+     * `created_at`, with a TTL of the inactivity destruction plus a day) and marks `e` recreated
+     * when the holder last used another instance. Best effort: a Redis error only logs.
+     */
+    private seenBy;
+    /** Runs `fn` as the holder `e`: its tag on every exec, its lease's signal. */
+    private runAs;
+    /**
+     * Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
+     * A sandbox recorded `running` (with the current provider, not the `gone` instance, and no legacy
+     * busy marker) is used under a shared lease; otherwise the exclusive lease (with intent) makes it
+     * run and is downgraded. One token serves both, so the waiter passes its own intent.
+     */
+    private enter;
+    private clearIntent;
+    private readRow;
+    /** Recorded running with the current provider, and not the instance found gone. */
+    private usable;
+    /**
+     * Under a just-taken shared lease: confirms the row is still usable (else releases and returns
+     * null), records the holder and kills crashed holders' commands.
+     */
+    private asShared;
+    /**
+     * Under a just-taken exclusive lease: makes the sandbox run (create / resume / rebuild when the
+     * recorded instance is the one found gone), kills leftovers, then downgrades to a shared lease.
+     */
+    private asExclusive;
+    /** Ends a holder: unregisters it before releasing its lease, then moves `last_used_at` forward. */
+    private leave;
+    private register;
+    private unregister;
+    /**
+     * Crashed holders' tags (claimed), and an older kernel's busy marker when `exclusiveToken` is
+     * the caller's exclusive lease; a Redis error counts as none.
+     */
+    private claim;
+    /** Runs one kill call: gone and abort are thrown, other failures only warned. */
+    private kill;
+    /**
+     * 休眠：空闲超过 `suspendAfterIdleMinutes`、或连续运行超过 `maxRunningMinutes` 的运行中沙箱。
+     * Skips a sandbox with any holder; one past `maxRunningMinutes` gets the intent for one poll
+     * window (about a second) first. 返回本次休眠的个数。由维护队列每分钟调用。
+     */
+    suspendIdle(now: Date): Promise<number>;
+    /**
+     * 销毁：超过 `destroyAfterInactiveDays` 没用过的、所属组织已删除的、对话已不存在的沙箱
+     * （对话已不存在的，销毁后删掉记录）。有 holder 的跳过。返回本次销毁的个数。由维护队列每小时调用。
+     */
+    destroyInactive(now: Date): Promise<number>;
+    /**
+     * 立即销毁该对话的沙箱（对话删除、离职处理时调用）；没有沙箱或已销毁时什么也不做。
+     * Waits for the exclusive lease (with intent, so running commands finish and new ones wait) up
+     * to 10 minutes. 没有提供方时只记 warn。
+     */
+    destroyFor(conversationId: string): Promise<void>;
+    /**
+     * 接手前任：杀掉沙箱里遗留的全部命令（killAll）。实例已不在（SandboxGoneError）与中止照常抛出（交给 withSandbox 重建 /
+     * 结束）；其余失败只记 warn。提供方没有 `killAll` 时只记 info。
+     */
+    private killLeftovers;
+    private require;
+    /**
+     * 防护写入：lease 仍在、且记录的 generation 仍是锁内读到的值时才写（并加一）；否则抛 {@link SandboxLockLostError}。
+     */
+    private fenced;
+    /** 扫描的候选：上次扫描失败的排在后面，其余按最久未用在前。 */
+    private candidates;
+    /**
+     * One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
+     * `maxRunningMinutes`) a busy sandbox gets the intent for about a second, so new commands wait
+     * briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
+     * `sweep_failed_at` (sorted last next time) and return false.
+     */
+    private tryLocked;
+    /**
+     * Under the exclusive lease: make the conversation's sandbox run. `gone`: the instance a holder
+     * found gone; when it is still the recorded one it is rebuilt, when another holder already
+     * rebuilt it the new one is used.
+     */
+    private ensure;
+    /**
+     * 在锁内创建：先防护地重置为 creating，再调提供方，最后防护地写入句柄。
+     * 写句柄时发现锁已丢失：尽力销毁刚建好的实例，再抛 {@link SandboxLockLostError}。
+     */
+    private create;
+    /**
+     * 用旧提供方销毁别的提供方创建的实例：旧提供方仍加载且销毁成功 → false；否则 warn 并返回 true（需要留着记录）。
+     */
+    private retire;
+    /** 在锁内：销毁实例（与遗留的旧提供方实例）；对话已不存在的删掉记录，否则标记 destroyed。 */
+    private destroyRow;
+    /** 该休眠的：本提供方创建、运行中，且空闲到期或连续运行超时。 */
+    private suspendDue;
+    /** 该销毁的：未销毁，且不活跃到期、组织已删除或对话已不存在。 */
+    private destroyDue;
+}