@capybara-hq/kernel 0.1.0-rc.3 → 0.1.0-rc.4

package/dist/background/stop.js

@@ -0,0 +1,225 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { Inject, Injectable } from "@nestjs/common";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { SandboxService } from "../sandbox/sandbox.service.js";
+import { BackgroundTaskFinisher, } from "./finish.js";
+import { TASKS_DIR } from "./launch.js";
+import { assertTaskRef, decodeTail, killSession, tailCommand } from "./session.js";
+import { BackgroundTaskStore } from "./store.js";
+/** How long a stop waits after SIGTERM before SIGKILL (spec §2.3). */
+export const STOP_GRACE_MS = 5000;
+/** The time limit of the SIGTERM exec: the grace period plus some room. */
+const TERM_EXEC_TIMEOUT_MS = STOP_GRACE_MS + 5000;
+/** The time limit of reading the last lines after a stop. */
+const TAIL_EXEC_TIMEOUT_MS = 10_000;
+/**
+ * Exit code 0 with `exited <code>` when the task had already finished (its exit file exists),
+ * else sends SIGTERM to the session and waits up to {@link STOP_GRACE_MS} for it to empty, then
+ * prints `term`. Relative to `/workspace`.
+ */
+function termScript(id, sid) {
+    const f = `${TASKS_DIR}/${id}`;
+    const tries = Math.ceil(STOP_GRACE_MS / 200);
+    return [
+        `if [ -s ${f}.exit ]; then echo "exited $(cat ${f}.exit)"; exit 0; fi`,
+        `pkill -TERM -s ${sid} >/dev/null 2>&1`,
+        `i=0; while [ "$i" -lt ${tries} ] && pgrep -s ${sid} >/dev/null 2>&1; do sleep 0.2; i=$((i+1)); done`,
+        "echo term",
+    ].join("\n");
+}
+/** How a task that exited by itself ends (spec §4.3): exit 137 past its deadline timed out. */
+export function exitedAs(task, code, now) {
+    if (code === 0)
+        return { status: "completed", exitCode: 0 };
+    if (code === 137 && now.getTime() >= task.deadlineAt.getTime()) {
+        return { status: "timed_out", exitCode: 137 };
+    }
+    return { status: "failed", exitCode: code };
+}
+/** How a task whose sandbox is not running ends (`ifRunning`'s `not_running` status). */
+export function notRunningAs(status, killPending) {
+    return {
+        status: "lost",
+        lostReason: status === "suspended" ? "sandbox_suspended" : "sandbox_gone",
+        killPending,
+    };
+}
+/** A task found in another sandbox instance (a rebuild): its processes are gone with the old one. */
+export const RECREATED = {
+    status: "lost",
+    lostReason: "sandbox_recreated",
+    killPending: false,
+};
+/**
+ * Stops background tasks (spec 2026-11-05-background-bash-design §2.3, §4.3): SIGTERM to the
+ * task's session, up to 5 s for it to empty, then SIGKILL until it is empty, all through
+ * `SandboxService.ifRunning` (it never creates or resumes a sandbox); the task then finishes as
+ * `stopped` with its last 20 lines. A task that had already exited finishes as its exit says. When
+ * the sandbox is not running the task finishes as `lost` with `kill_pending` (killed right after a
+ * resume); in a rebuilt sandbox as `lost` (`sandbox_recreated`). Processes a task that had
+ * already exited left running are killed as well.
+ */
+let BackgroundTaskStopper = class BackgroundTaskStopper {
+    store;
+    finisher;
+    sandbox;
+    db;
+    log;
+    constructor(store, finisher, sandbox, db, log) {
+        this.store = store;
+        this.finisher = finisher;
+        this.sandbox = sandbox;
+        this.db = db;
+        this.log = log;
+    }
+    /**
+     * Stops `task` and returns its row afterwards. A task that is no longer running, or that another
+     * live stop already claimed, is returned as it is now.
+     *
+     * The stop first claims the task (`stopping_at`), so the poller leaves it alone while the
+     * signals take effect and the stop's finish is its only one. A reservation whose launch has not
+     * started the command yet (no `sid`) is only claimed: the launch then kills what it starts and
+     * finishes the row `stopped`. When the stop fails while the sandbox runs (busy, a provider
+     * error, a session that could not be confirmed empty), the claim is given up, the task stays
+     * `running` and the error is thrown.
+     */
+    async stop(task, why, signal) {
+        if (!(await this.store.claimStop(task.id)))
+            return (await this.store.get(task.id)) ?? task;
+        // read after the claim: a launch recording its session before the claim is seen here, one
+        // recording it after sees the claim (both are single-row updates of the same row)
+        const cur = (await this.store.get(task.id)) ?? task;
+        if (cur.status !== "running" || cur.sid === null || cur.sandboxCreatedAt === null)
+            return cur;
+        const sid = cur.sid;
+        assertTaskRef(cur.id, sid);
+        const reported = why === "tool";
+        // a person's stop never wakes the conversation (background Bash §3, fix round 1), nor does
+        // the stop of a conversation being deleted
+        const stoppedBy = why === "person" ? "person" : why === "conversation_deleted" ? "deletion" : undefined;
+        const opts = stoppedBy === undefined ? {} : { stoppedBy };
+        let r;
+        try {
+            r = await this.sandbox.ifRunning(cur.sandboxConversationId, { createdAt: cur.sandboxCreatedAt }, async (p, h, sig) => {
+                const term = await p.exec(h, {
+                    command: termScript(cur.id, sid),
+                    cwd: "/workspace",
+                    timeoutMs: TERM_EXEC_TIMEOUT_MS,
+                    maxOutputBytes: 4096,
+                    signal: sig,
+                });
+                const exited = /^exited (-?[0-9]+)$/.exec(term.stdout.trim());
+                if (term.exitCode !== 0 || (!exited && term.stdout.trim() !== "term")) {
+                    throw new Error(`the stop of background task ${cur.id} failed (${term.exitCode})`);
+                }
+                // also after an exit: processes the command left running are stopped with the task
+                if (!(await killSession(p, h, sid))) {
+                    throw new Error(`background task ${cur.id} could not be confirmed stopped`);
+                }
+                const tail = await p.exec(h, {
+                    command: tailCommand(cur.id),
+                    cwd: "/workspace",
+                    timeoutMs: TAIL_EXEC_TIMEOUT_MS,
+                    maxOutputBytes: 64 * 1024,
+                    signal: sig,
+                });
+                const lastLines = tail.exitCode === 0 ? decodeTail(tail.stdout) : "";
+                // a task that had exited before the signal ends as its exit says
+                const to = exited
+                    ? exitedAs(cur, Number(exited[1]), new Date())
+                    : { status: "stopped" };
+                return { to, lastLines };
+            }, signal);
+        }
+        catch (err) {
+            await this.store
+                .releaseStop(cur.id)
+                .catch((e) => this.log.warn("cannot release a background task stop", { taskId: cur.id, err: e }));
+            throw err;
+        }
+        if (r.kind === "ran") {
+            await this.finisher.finish(cur, r.value.to, r.value.lastLines, reported, opts);
+        }
+        else if (r.kind === "not_running") {
+            await this.finisher.finish(cur, notRunningAs(r.status, true), "", reported, opts);
+        }
+        else {
+            await this.finisher.finish(cur, RECREATED, "", reported, opts);
+        }
+        return (await this.store.get(cur.id)) ?? cur;
+    }
+    /**
+     * After the sub-agent run `runId` ended (spec §4.3): stops the running tasks that run started in
+     * `conversationId` when it is a sub-agent's conversation (nobody can read or stop them through
+     * that run's model after that); any other conversation is left alone. Only that run's tasks: a
+     * follow-up run of the same child keeps its own when an older run's end is handled late. Never
+     * throws (see {@link stopAll}).
+     */
+    async stopSubagentTasks(conversationId, runId) {
+        try {
+            const conv = await this.db
+                .selectFrom("conversations")
+                .select("kind")
+                .where("id", "=", conversationId)
+                .executeTakeFirst();
+            if (conv?.kind !== "subagent")
+                return;
+            const tasks = (await this.store.runningOwnedBy(conversationId)).filter((t) => t.runId === runId);
+            await this.stopAll(tasks, "subagent_end");
+        }
+        catch (err) {
+            this.log.warn("cannot stop the background tasks of a finished sub-agent run", {
+                conversationId,
+                runId,
+                err,
+            });
+        }
+    }
+    /**
+     * Before a conversation is deleted: stops every running task in its sandbox (its own and its
+     * sub-agents'). Never throws (see {@link stopAll}).
+     */
+    async stopForDeletedConversation(conversationId) {
+        try {
+            await this.stopAll(await this.store.runningIn(conversationId), "conversation_deleted");
+        }
+        catch (err) {
+            this.log.warn("cannot stop the background tasks of a deleted conversation", {
+                conversationId,
+                err,
+            });
+        }
+    }
+    /**
+     * Stops `tasks` side by side. A stop that fails while the sandbox runs leaves its task running
+     * (logged): the poller still checks it and it ends at its own deadline at the latest.
+     */
+    async stopAll(tasks, why) {
+        await Promise.all(tasks.map(async (t) => {
+            try {
+                await this.stop(t, why);
+            }
+            catch (err) {
+                this.log.warn("cannot stop a background task; it keeps running", { taskId: t.id, err });
+            }
+        }));
+    }
+};
+BackgroundTaskStopper = __decorate([
+    Injectable(),
+    __param(0, Inject(BackgroundTaskStore)),
+    __param(1, Inject(BackgroundTaskFinisher)),
+    __param(2, Inject(SandboxService)),
+    __param(3, Inject(KernelDb)),
+    __param(4, Inject(KERNEL_LOGGER))
+], BackgroundTaskStopper);
+export { BackgroundTaskStopper };

package/dist/background/store.d.ts

@@ -0,0 +1,150 @@
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { BackgroundLaunchLock } from "./launch-lock.js";
+export type BackgroundTaskStatus = "running" | "completed" | "failed" | "stopped" | "timed_out" | "lost";
+/** A task as the kernel reads it: the columns of `capybara.background_tasks`, camel-cased. */
+export interface BackgroundTaskRow {
+    id: string;
+    orgId: string;
+    /** The owner: the child conversation for a sub-agent. */
+    conversationId: string;
+    /** The root conversation that owns the sandbox. */
+    sandboxConversationId: string;
+    runId: string;
+    callId: string;
+    triggerAccountId: string;
+    description: string;
+    command: string;
+    /**
+     * The session id (the pid `setsid` started). Null while the launch is reserved but has not
+     * started: readers that act on the process (the poller, stops) skip such a row.
+     */
+    sid: string | null;
+    /** Relative to `/workspace`. */
+    outputPath: string;
+    status: BackgroundTaskStatus;
+    lostReason: string | null;
+    exitCode: number | null;
+    startedAt: Date;
+    finishedAt: Date | null;
+    deadlineAt: Date;
+    /** The sandbox instance the task started in; null while reserved, like `sid`. */
+    sandboxCreatedAt: Date | null;
+    /** Marked lost while its process may still exist. */
+    killPending: boolean;
+    /** The bytes `TaskOutput` has already returned. */
+    readOffset: number;
+    /** When a stop claimed the task ({@link BackgroundTaskStore.claimStop}); null otherwise. */
+    stoppingAt: Date | null;
+}
+export type BackgroundTaskInsert = Omit<BackgroundTaskRow, "status" | "finishedAt" | "exitCode" | "lostReason" | "killPending" | "readOffset" | "stoppingAt">;
+/** A launch's reservation: the row before the command started (no `sid`, no instance yet). */
+export type BackgroundTaskReservation = Omit<BackgroundTaskInsert, "sid" | "sandboxCreatedAt">;
+/**
+ * How long a stop's claim holds: a claim older than this is abandoned (its stopper died), so the
+ * poller checks the task again and another stop may claim it. Longer than any stop takes (the
+ * lease wait, the SIGTERM grace, the kill and the tail).
+ */
+export declare const STOP_CLAIM_TTL_MS: number;
+/**
+ * Background Bash tasks (spec §4.1): one `capybara.background_tasks` row per command started with
+ * `run_in_background`. The store only reads and writes rows; the launch, the poller and the tools
+ * that drive them are other providers.
+ *
+ * A launch first reserves its row ({@link BackgroundTaskStore.reserve}: `running`, `sid` null),
+ * then starts the command and sets `sid` ({@link BackgroundTaskStore.markStarted}), or finishes
+ * the row as `failed` ({@link BackgroundTaskStore.failLaunch}). A running row with a null `sid`
+ * counts against `maxTasks` but has no process to check or stop: the poller and stops must skip
+ * it (the launch that reserved it settles it; one left over by a launch that died is failed by the
+ * poller, {@link BackgroundTaskStore.failAbandonedLaunches}).
+ */
+export declare class BackgroundTaskStore {
+    private readonly db;
+    readonly txHost: TransactionHost;
+    private readonly launchLock;
+    constructor(db: KernelDb, txHost: TransactionHost, launchLock: BackgroundLaunchLock);
+    /**
+     * Reserves a launch: under the sandbox's launch lock, counts the running tasks of
+     * `row.sandboxConversationId` and inserts `row` as `running` (no `sid`) when fewer than
+     * `maxTasks` run. False (nothing written) when the sandbox already runs `maxTasks`.
+     */
+    reserve(row: BackgroundTaskReservation, maxTasks: number): Promise<boolean>;
+    /**
+     * Records the started command of a reserved row: `started`, or `stopping` when a stop claimed the
+     * reservation meanwhile (the launch then kills the command and finishes the row with
+     * {@link finishStoppedLaunch}); false when the row is no longer running.
+     */
+    markStarted(id: string, to: {
+        sid: string;
+        sandboxCreatedAt: Date;
+    }): Promise<"started" | "stopping" | false>;
+    /**
+     * Finishes a launch whose reservation a stop claimed, once the launch killed what it started:
+     * `stopped`. No `background_task_finished` event: the launching call reports it itself.
+     */
+    finishStoppedLaunch(id: string): Promise<void>;
+    /**
+     * Claims a running task for a stop (`stopping_at`): false when it is no longer running or another
+     * live stop claimed it ({@link STOP_CLAIM_TTL_MS}). While claimed the poller leaves it alone.
+     */
+    claimStop(id: string): Promise<boolean>;
+    /** Gives up a stop's claim (the stop failed while the sandbox runs: the task keeps running). */
+    releaseStop(id: string): Promise<void>;
+    /**
+     * Finishes a reserved row whose launch failed as `failed`, with `reason` in `lost_reason` (and a
+     * stop's claim cleared); a row that already has its sid (the command started and is tracked) is
+     * left as it is. No `background_task_finished` event: the launching call reports the failure
+     * itself.
+     */
+    failLaunch(id: string, reason: string): Promise<void>;
+    /**
+     * Finishes as `failed` (`lost_reason` `launch_abandoned`) every reservation that started before
+     * `before`, never got its sid and is not claimed by a live stop: the launch that reserved it died
+     * (a crashed replica). A stop's claim older than {@link STOP_CLAIM_TTL_MS} does not hold it: a
+     * stop only claims a reservation and leaves the rest to the launch, so a dead launch would keep
+     * it (and its slot of `maxTasks`) forever. Conditional on `sid` still null, so a launch
+     * completing at that moment wins. No `background_task_finished` event: the command never
+     * started and the model never saw its id. Returns the rows it finished (their owner and sandbox
+     * conversations, for the caller's `changed: "background"` frames).
+     */
+    failAbandonedLaunches(before: Date): Promise<{
+        id: string;
+        conversationId: string;
+        sandboxConversationId: string;
+    }[]>;
+    insert(row: BackgroundTaskInsert): Promise<void>;
+    get(id: string): Promise<BackgroundTaskRow | undefined>;
+    /** The running tasks of one sandbox, oldest first. */
+    runningIn(sandboxConversationId: string): Promise<BackgroundTaskRow[]>;
+    /**
+     * Every running task that has started (its `sid` is set) and no live stop claimed, across
+     * sandboxes, oldest first: what the poller checks. A reservation still launching (`sid` null) is
+     * left to its launch, a claimed task to its stop.
+     */
+    runningStarted(): Promise<BackgroundTaskRow[]>;
+    /** The running tasks one conversation owns (not those of its sub-agents), oldest first. */
+    runningOwnedBy(conversationId: string): Promise<BackgroundTaskRow[]>;
+    countRunning(sandboxConversationId: string): Promise<number>;
+    /** The tasks owned by any of the conversations, newest first. */
+    listFor(conversationIds: string[]): Promise<BackgroundTaskRow[]>;
+    /**
+     * Conditional `running` to a final status; false when the row was no longer running (another
+     * path finished it first), or, with `opts.unclaimed` (the poller), when a live stop claimed it.
+     * The caller's transaction also appends the finished event.
+     */
+    finish(id: string, to: {
+        status: Exclude<BackgroundTaskStatus, "running">;
+        exitCode?: number | null;
+        lostReason?: string;
+        killPending?: boolean;
+    }, opts?: {
+        unclaimed?: boolean;
+    }): Promise<boolean>;
+    /** Moves the read offset forward only (a concurrent reader never moves it back). */
+    advanceOffset(id: string, to: number): Promise<void>;
+    /** Finished tasks of a sandbox whose process may still exist. */
+    pendingKills(sandboxConversationId: string): Promise<BackgroundTaskRow[]>;
+    clearKill(ids: string[]): Promise<void>;
+    /** Deletes finished tasks that ended before `before`; a running one is never deleted. */
+    pruneFinished(before: Date): Promise<number>;
+}