@sema-agent/server 7.147.0 → 7.149.0

package/dist/plugins/run-store-sql.js

@@ -1,25 +1,36 @@
-import { redactErrorMessage, redactLedgerEventData, sealRunRowResult } from "../observability/run-terminal.js";
-import { outwardPlane } from "../terminal.js";
+import { redactErrorMessage, redactLedgerEventData, runRowErrorCodeOf, sealRunRowResult } from "../observability/run-terminal.js";
+import { outwardPlane, persistedPlaneOf } from "../terminal.js";
 import { projectUsageStats, USAGE_SCAN_LIMIT } from "../usage-analytics.js";
-import { escapeLike } from "./sql-escape.js";
 import { pgSanitizeText } from "./pg-safe-json.js";
 import { toIso as iso } from "./sql-row-helpers.js";
 import { recordFailOpen } from "../observability/fail-open.js";
 import { createObjectColumnReader, decodeJsonColumn, decodeNullableJsonColumn, foldJsonColumn, isJsonColumnBodyProblem, jsonColumnText, readJsonObjectColumn, readNonNullJsonValueColumn } from "./sql-json-column.js";
-import { RUN_HOLDER_DEATH_CODES, RUN_STALLED_ERROR, settledLegObservationOf } from "./store-contracts.js";
+import { RUN_HOLDER_DEATH_CODES, RUN_STALLED_ERROR, RUN_STALL_DEFAULT_WIRING, RUN_STALL_VERDICT_UNLANDED_TAG, isParkedRunStatus, isTerminalRunStatus, mintRunLegId, runLivenessOf, settledLegObservationOf } from "./store-contracts.js";
 import { notifyRunTerminal, runTerminalLogLevel } from "../observability/run-terminal.js";
 import { mysqlDriver, pgDriver, dialectJsonEncoder, SQL_DB_NOW_MS, sqlDbNowMs } from "./sql-driver.js";
 import { isDupKeyError } from "./sql-errors.js";
+import { readDeletingTokenOn } from "./session-ext-table.js";
 import { readRunIdentityCells, runIdentityCells } from "../identity-source.js";
 import { decideSpaceCommitAbandon, decideSpaceCommitClaim, fenceSpaceCommitAtTerminal, spaceCommitStateOf } from "../workspace-space-commit.js";
 function runRowCols(dialect) {
     return (`task_id, session_id, owner, status, ${jsonColumnText(dialect, "result")}, error, error_code, job_id, source, ` +
-        "identity_source, identity_scope, identity_principal_inc, instance_id, cancel_requested, objective_preview, created_at, updated_at");
+        "identity_source, identity_scope, identity_principal_inc, instance_id, leg_id, cancel_requested, objective_preview, created_at, updated_at");
 }
 const readPersistedTaskResultBody = createObjectColumnReader();
 function decodeResultColumn(v, taskId) {
     return decodeNullableJsonColumn(v, readPersistedTaskResultBody, `task_run.result for run ${taskId}`);
 }
+export function decodeRunResultCellM44(v, taskId) {
+    try {
+        return decodeResultColumn(v, taskId) ?? null;
+    }
+    catch (e) {
+        if (!isJsonColumnBodyProblem(e))
+            throw e;
+        recordFailOpen("server.run-store.m44-result-unreadable", e.message);
+        return "unreadable";
+    }
+}
 function refusing(what, read) {
     try {
         return read();
@@ -42,10 +53,19 @@ function usageResultBody(v) {
 function decodeEventDataColumn(v, taskId, seq) {
     return decodeNullableJsonColumn(v, readNonNullJsonValueColumn, `task_event.data for run ${taskId} seq ${seq}`);
 }
+function m44ErrorCodeReadSide(status, result, column) {
+    if (status === "running")
+        return null;
+    if (column !== null || !isParkedRunStatus(status))
+        return column;
+    return persistedPlaneOf(result).errorCode ?? null;
+}
 export class SqlRunStore {
     db;
-    constructor(db) {
+    stall;
+    constructor(db, stall = RUN_STALL_DEFAULT_WIRING) {
         this.db = db;
+        this.stall = stall;
     }
     q(mysql, pg) {
         return this.db.dialect === "mysql" ? mysql : pg;
@@ -58,6 +78,14 @@ export class SqlRunStore {
     }
     async tx(fn) {
         const conn = await this.db.connect();
+        try {
+            await this.txOn(conn, fn);
+        }
+        finally {
+            conn.release();
+        }
+    }
+    async txOn(conn, fn) {
         try {
             await conn.begin();
             await fn(conn);
@@ -71,18 +99,27 @@ export class SqlRunStore {
             }
             throw err;
         }
+    }
+    async createRun(taskId, sessionId, owner, instanceId, meta = {}) {
+        const legId = mintRunLegId();
+        const conn = await this.db.connect();
+        try {
+            const first = await this.createRunOn(conn, taskId, sessionId, owner, instanceId, legId, meta);
+            if (first.activeTaskId === undefined || !(await this.settleDueHolderOn(conn, first.activeTaskId)))
+                return first;
+            return await this.createRunOn(conn, taskId, sessionId, owner, instanceId, legId, meta);
+        }
         finally {
             conn.release();
         }
     }
-    async createRun(taskId, sessionId, owner, instanceId, meta = {}) {
+    async createRunOn(conn, taskId, sessionId, owner, instanceId, legId, meta) {
         const idemKey = meta.idem?.key ?? null;
-        const conn = await this.db.connect();
         let stage = "claim";
         try {
             await conn.begin();
             const { rows: root } = await conn.query(this.q("SELECT session_id FROM session_meta WHERE session_id = ? FOR UPDATE", "SELECT session_id FROM session_meta WHERE session_id = $1 FOR UPDATE"), [sessionId]);
-            if (root.length === 0) {
+            if (root.length === 0 || (await readDeletingTokenOn(async (sql, values) => (await conn.query(sql, [...values])).rows, this.db.dialect, sessionId)) !== null) {
                 await conn.rollback();
                 return { ok: false, sessionAbsent: true };
             }
@@ -91,13 +128,14 @@ export class SqlRunStore {
                 await conn.query(this.q("INSERT INTO task_active (session_id, task_id, since) VALUES (?,?,?)", "INSERT INTO task_active (session_id, task_id, since) VALUES ($1,$2,$3)"), [sessionId, taskId, now]);
                 stage = "row";
                 const cells = runIdentityCells(meta.identity ?? null);
-                await conn.query(this.q("INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
-                    "VALUES (?,?,?,'running',?,?,?,?,?,?,?,?,?,?,?)", "INSERT INTO task_run (task_id, session_id, owner, status, instance_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
-                    "VALUES ($1,$2,$3,'running',$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)"), [
+                await conn.query(this.q("INSERT INTO task_run (task_id, session_id, owner, status, instance_id, leg_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
+                    "VALUES (?,?,?,'running',?,?,?,?,?,?,?,?,?,?,?,?)", "INSERT INTO task_run (task_id, session_id, owner, status, instance_id, leg_id, job_id, source, identity_source, identity_scope, identity_principal_inc, objective_preview, idem_key, idem_body_fingerprint, created_at, updated_at) " +
+                    "VALUES ($1,$2,$3,'running',$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)"), [
                     taskId,
                     sessionId,
                     owner,
                     instanceId,
+                    legId,
                     meta.jobId ?? null,
                     meta.source ?? null,
                     cells.identitySource,
@@ -114,7 +152,7 @@ export class SqlRunStore {
                     now,
                 ]);
                 await conn.commit();
-                return { ok: true };
+                return { ok: true, legId };
             }
             catch (err) {
                 if (!this.isDupKey(err))
@@ -137,9 +175,62 @@ export class SqlRunStore {
             }
             throw err;
         }
-        finally {
-            conn.release();
+    }
+    async settleDueHolderOn(conn, holderTaskId) {
+        const { rows } = await conn.query(this.q("SELECT status, updated_at FROM task_run WHERE task_id = ?", "SELECT status, updated_at FROM task_run WHERE task_id = $1"), [holderTaskId]);
+        const row = rows[0];
+        if (row === undefined)
+            return false;
+        const nowMs = Date.now();
+        const windowMs = this.stall.windowMs();
+        if (runLivenessOf({ status: String(row.status), updatedAt: iso(row.updated_at) }, nowMs, windowMs) !== "due")
+            return false;
+        let landed = 0;
+        try {
+            await this.txOn(conn, async (c) => {
+                landed = await this.landStallVerdicts(c, new Date(nowMs - windowMs), holderTaskId);
+            });
+        }
+        catch (err) {
+            recordFailOpen(RUN_STALL_VERDICT_UNLANDED_TAG, err instanceof Error ? err.message : String(err));
+            return false;
         }
+        if (landed > 0)
+            this.stall.onVerdicts(landed);
+        return landed > 0;
+    }
+    async landStallVerdicts(conn, cutoff, taskId) {
+        const one = taskId !== undefined;
+        const { affected } = await conn.query(this.q(`UPDATE task_run SET status = 'failed', result = NULL, error = '${RUN_STALLED_ERROR}', error_code = '${RUN_HOLDER_DEATH_CODES.stalled}', updated_at = ? ` +
+            `WHERE status = 'running' AND updated_at < ?${one ? " AND task_id = ?" : ""}`, `UPDATE task_run SET status = 'failed', result = NULL, error = '${RUN_STALLED_ERROR}', error_code = '${RUN_HOLDER_DEATH_CODES.stalled}', updated_at = $1 ` +
+            `WHERE status = 'running' AND updated_at < $2${one ? " AND task_id = $3" : ""}`), one ? [new Date(), cutoff, taskId] : [new Date(), cutoff]);
+        if (one) {
+            if (affected === 1)
+                await conn.query(this.q("DELETE FROM task_active WHERE task_id = ?", "DELETE FROM task_active WHERE task_id = $1"), [taskId]);
+            return affected;
+        }
+        await conn.query(this.q("DELETE ta FROM task_active ta JOIN task_run tr ON ta.task_id = tr.task_id WHERE tr.status NOT IN ('running','suspended','needs_review')", "DELETE FROM task_active ta USING task_run tr WHERE ta.task_id = tr.task_id AND tr.status NOT IN ('running','suspended','needs_review')"));
+        await conn.query(this.q("DELETE FROM task_active WHERE NOT EXISTS (SELECT 1 FROM task_run tr WHERE tr.task_id = task_active.task_id)", "DELETE FROM task_active WHERE NOT EXISTS (SELECT 1 FROM task_run tr WHERE tr.task_id = task_active.task_id)"));
+        return affected;
+    }
+    async observeRun(row) {
+        const nowMs = Date.now();
+        const windowMs = this.stall.windowMs();
+        if (runLivenessOf(row, nowMs, windowMs) !== "due")
+            return row;
+        let landed = 0;
+        try {
+            await this.tx(async (conn) => {
+                landed = await this.landStallVerdicts(conn, new Date(nowMs - windowMs), row.taskId);
+            });
+        }
+        catch (err) {
+            recordFailOpen(RUN_STALL_VERDICT_UNLANDED_TAG, err instanceof Error ? err.message : String(err));
+            return row;
+        }
+        if (landed > 0)
+            this.stall.onVerdicts(landed);
+        return (await this.getRun(row.taskId)) ?? row;
     }
     async fillObjectivePreview(taskId, preview) {
         await this.db.query(this.q("UPDATE task_run SET objective_preview = ? WHERE task_id = ? AND objective_preview IS NULL", "UPDATE task_run SET objective_preview = $1 WHERE task_id = $2 AND objective_preview IS NULL"), [this.db.dialect === "mysql" ? preview : pgSanitizeText(preview), taskId]);
@@ -161,7 +252,11 @@ export class SqlRunStore {
         return Boolean(rows[0]?.preempt_requested);
     }
     async heartbeat(taskId, owner) {
-        await this.db.query(this.q("UPDATE task_run SET updated_at = ? WHERE task_id = ? AND owner <=> ? AND status = 'running'", "UPDATE task_run SET updated_at = $1 WHERE task_id = $2 AND owner IS NOT DISTINCT FROM $3 AND status = 'running'"), [new Date(), taskId, owner]);
+        const { affected } = await this.db.query(this.q("UPDATE task_run SET updated_at = ? WHERE task_id = ? AND owner <=> ? AND status = 'running'", "UPDATE task_run SET updated_at = $1 WHERE task_id = $2 AND owner IS NOT DISTINCT FROM $3 AND status = 'running'"), [new Date(), taskId, owner]);
+        if (affected > 0)
+            return true;
+        const { rows } = await this.db.query(this.q("SELECT status FROM task_run WHERE task_id = ?", "SELECT status FROM task_run WHERE task_id = $1"), [taskId]);
+        return rows[0] !== undefined && String(rows[0].status) === "running";
     }
     async appendEvent(taskId, seq, type, data) {
         data = redactLedgerEventData(type, data);
@@ -175,7 +270,7 @@ export class SqlRunStore {
     async setTerminal(taskId, status, result, error) {
         result = sealRunRowResult(result);
         error = error === null ? null : (redactErrorMessage(error) ?? null);
-        const errorCode = result === null ? null : (outwardPlane(result.terminal).errorCode ?? null);
+        const errorCode = runRowErrorCodeOf(result);
         let won = false;
         let observed;
         await fenceSpaceCommitAtTerminal(taskId, async (spaceCommit) => {
@@ -270,17 +365,20 @@ export class SqlRunStore {
             }
         }
     }
-    async setSuspended(taskId, result) {
+    async setSuspended(taskId, result, legId) {
         const sealed = sealRunRowResult(result);
-        await this.db.query(this.q("UPDATE task_run SET status = 'suspended', result = ?, updated_at = ? WHERE task_id = ? AND status IN ('running','suspended')", "UPDATE task_run SET status = 'suspended', result = $1::jsonb, updated_at = $2 WHERE task_id = $3 AND status IN ('running','suspended')"), [sealed == null ? null : this.json(sealed), new Date(), taskId]);
+        const { affected } = await this.db.query(this.q("UPDATE task_run SET status = 'suspended', result = ?, error_code = ?, updated_at = ? WHERE task_id = ? AND leg_id = ? AND status IN ('running','suspended') AND cancel_requested = 0", "UPDATE task_run SET status = 'suspended', result = $1::jsonb, error_code = $2, updated_at = $3 WHERE task_id = $4 AND leg_id = $5 AND status IN ('running','suspended') AND cancel_requested = 0"), [sealed == null ? null : this.json(sealed), runRowErrorCodeOf(sealed), new Date(), taskId, legId]);
+        return affected > 0;
     }
-    async setNeedsReview(taskId, result) {
+    async setNeedsReview(taskId, result, legId) {
         const sealed = sealRunRowResult(result);
-        await this.db.query(this.q("UPDATE task_run SET status = 'needs_review', result = ?, updated_at = ? WHERE task_id = ? AND status IN ('running','needs_review')", "UPDATE task_run SET status = 'needs_review', result = $1::jsonb, updated_at = $2 WHERE task_id = $3 AND status IN ('running','needs_review')"), [sealed == null ? null : this.json(sealed), new Date(), taskId]);
+        const { affected } = await this.db.query(this.q("UPDATE task_run SET status = 'needs_review', result = ?, error_code = ?, updated_at = ? WHERE task_id = ? AND leg_id = ? AND status IN ('running','needs_review') AND cancel_requested = 0", "UPDATE task_run SET status = 'needs_review', result = $1::jsonb, error_code = $2, updated_at = $3 WHERE task_id = $4 AND leg_id = $5 AND status IN ('running','needs_review') AND cancel_requested = 0"), [sealed == null ? null : this.json(sealed), runRowErrorCodeOf(sealed), new Date(), taskId, legId]);
+        return affected > 0;
     }
     async markResuming(taskId, instanceId) {
-        const { affected } = await this.db.query(this.q("UPDATE task_run SET status = 'running', cancel_requested = 0, preempt_requested = 0, result = NULL, instance_id = ?, updated_at = ? WHERE task_id = ? AND status IN ('suspended','needs_review')", "UPDATE task_run SET status = 'running', cancel_requested = 0, preempt_requested = 0, result = NULL, instance_id = $1, updated_at = $2 WHERE task_id = $3 AND status IN ('suspended','needs_review')"), [instanceId, new Date(), taskId]);
-        return affected === 1;
+        const legId = mintRunLegId();
+        const { affected } = await this.db.query(this.q("UPDATE task_run SET status = 'running', cancel_requested = 0, preempt_requested = 0, result = NULL, error_code = NULL, instance_id = ?, leg_id = ?, updated_at = ? WHERE task_id = ? AND status IN ('suspended','needs_review')", "UPDATE task_run SET status = 'running', cancel_requested = 0, preempt_requested = 0, result = NULL, error_code = NULL, instance_id = $1, leg_id = $2, updated_at = $3 WHERE task_id = $4 AND status IN ('suspended','needs_review')"), [instanceId, legId, new Date(), taskId]);
+        return affected === 1 ? legId : undefined;
     }
     async whileHoldingClaim(sessionId, taskId, write) {
         let fenced = { held: false };
@@ -303,18 +401,20 @@ export class SqlRunStore {
         return rows[0] ? String(rows[0].task_id) : undefined;
     }
     rowToRun(r) {
+        const result = refusing("serve a finished run as if it had produced nothing", () => decodeResultColumn(r.result, String(r.task_id))) ?? null;
         return {
             taskId: String(r.task_id),
             sessionId: String(r.session_id),
             owner: r.owner ?? null,
             status: r.status,
-            result: refusing("serve a finished run as if it had produced nothing", () => decodeResultColumn(r.result, String(r.task_id))) ?? null,
+            result,
             error: r.error ?? null,
-            errorCode: r.error_code ?? null,
+            errorCode: m44ErrorCodeReadSide(String(r.status), result, r.error_code ?? null),
             jobId: r.job_id ?? null,
             source: r.source ?? null,
             identity: readRunIdentityCells(r.identity_source, r.identity_scope, r.identity_principal_inc),
             instanceId: r.instance_id ?? null,
+            legId: r.leg_id ?? null,
             cancelRequested: Boolean(r.cancel_requested),
             objectivePreview: r.objective_preview ?? null,
             createdAt: iso(r.created_at),
@@ -339,7 +439,29 @@ export class SqlRunStore {
         return rows[0] ? this.rowToRun(rows[0]) : undefined;
     }
     async releaseIdemKey(taskId) {
-        await this.db.query(this.q("UPDATE task_run SET idem_key = NULL, idem_body_fingerprint = NULL WHERE task_id = ?", "UPDATE task_run SET idem_key = NULL, idem_body_fingerprint = NULL WHERE task_id = $1"), [taskId]);
+        await this.db.query(this.releaseIdemKeySql(), [taskId]);
+    }
+    releaseIdemKeySql() {
+        return this.q("UPDATE task_run SET idem_key = NULL, idem_body_fingerprint = NULL WHERE task_id = ?", "UPDATE task_run SET idem_key = NULL, idem_body_fingerprint = NULL WHERE task_id = $1");
+    }
+    async withdrawMintedRun(taskId, legId) {
+        let withdrawn = false;
+        await this.tx(async (conn) => {
+            const { rows } = await conn.query(this.q("SELECT status FROM task_run WHERE task_id = ? AND leg_id = ? FOR UPDATE", "SELECT status FROM task_run WHERE task_id = $1 AND leg_id = $2 FOR UPDATE"), [taskId, legId]);
+            if (rows.length === 0)
+                return;
+            const status = rows[0].status;
+            if (status !== "running") {
+                if (isTerminalRunStatus(status))
+                    await conn.query(this.releaseIdemKeySql(), [taskId]);
+                return;
+            }
+            await conn.query(this.q("DELETE FROM task_event WHERE task_id = ?", "DELETE FROM task_event WHERE task_id = $1"), [taskId]);
+            await conn.query(this.q("DELETE FROM task_active WHERE task_id = ?", "DELETE FROM task_active WHERE task_id = $1"), [taskId]);
+            await conn.query(this.q("DELETE FROM task_run WHERE task_id = ?", "DELETE FROM task_run WHERE task_id = $1"), [taskId]);
+            withdrawn = true;
+        });
+        return withdrawn;
     }
     async bindIdemBodyFingerprint(taskId, idem) {
         await this.db.query(this.q("UPDATE task_run SET idem_body_fingerprint = ? WHERE task_id = ? AND idem_key = ? AND idem_body_fingerprint IS NULL", "UPDATE task_run SET idem_body_fingerprint = $1 WHERE task_id = $2 AND idem_key = $3 AND idem_body_fingerprint IS NULL"), [idem.bodyFingerprint, taskId, idem.key]);
@@ -424,89 +546,9 @@ export class SqlRunStore {
         const taskId = String(r.task_id);
         return settledLegObservationOf(taskId, refusing("read a settled leg as if it had observed no memory", () => decodeResultColumn(r.result, taskId)) ?? null);
     }
-    async listSessions(opts) {
-        if (this.db.dialect === "mysql") {
-            const params = [];
-            let innerWhere = "";
-            if (opts.owner !== undefined) {
-                innerWhere = opts.includeUnowned ? " WHERE (owner = ? OR owner IS NULL)" : " WHERE owner = ?";
-                params.push(opts.owner);
-            }
-            const outer = ["rn = 1"];
-            if (opts.q) {
-                outer.push("EXISTS (SELECT 1 FROM task_run tr WHERE tr.session_id = t.session_id AND LOWER(tr.objective_preview) LIKE LOWER(?) ESCAPE '\\\\')");
-                params.push(`%${escapeLike(opts.q)}%`);
-            }
-            if (opts.cursor) {
-                outer.push("(last_activity < ? OR (last_activity = ? AND session_id < ?))");
-                params.push(new Date(opts.cursor.lastActivityAt), new Date(opts.cursor.lastActivityAt), opts.cursor.sessionId);
-            }
-            const sql = "SELECT session_id, owner, task_id, last_activity, first_activity, run_count, objective_preview, status, last_task_run_id FROM (" +
-                "SELECT session_id, owner, task_id, objective_preview, status, JSON_UNQUOTE(JSON_EXTRACT(result, '$.runId')) AS last_task_run_id," +
-                " ROW_NUMBER() OVER (PARTITION BY session_id ORDER BY created_at DESC, task_id DESC) AS rn," +
-                " MAX(created_at) OVER (PARTITION BY session_id) AS last_activity," +
-                " MIN(created_at) OVER (PARTITION BY session_id) AS first_activity," +
-                " COUNT(*) OVER (PARTITION BY session_id) AS run_count" +
-                " FROM task_run" + innerWhere + ") t WHERE " + outer.join(" AND ") +
-                " ORDER BY last_activity DESC, session_id DESC LIMIT ?";
-            params.push(opts.limit);
-            const { rows } = await this.db.query(sql, params);
-            return rows.map((r) => this.rowToSession(r));
-        }
-        const params = [];
-        const p = (v) => {
-            params.push(v);
-            return `$${params.length}`;
-        };
-        const innerWhere = opts.owner !== undefined ? (opts.includeUnowned ? ` WHERE (owner = ${p(opts.owner)} OR owner IS NULL)` : ` WHERE owner = ${p(opts.owner)}`) : "";
-        const outer = ["rn = 1"];
-        if (opts.q)
-            outer.push(`EXISTS (SELECT 1 FROM task_run tr WHERE tr.session_id = t.session_id AND LOWER(tr.objective_preview) LIKE LOWER(${p(`%${escapeLike(opts.q)}%`)}) ESCAPE '\\')`);
-        if (opts.cursor) {
-            const a = p(new Date(opts.cursor.lastActivityAt));
-            const b = p(new Date(opts.cursor.lastActivityAt));
-            const sid = p(opts.cursor.sessionId);
-            outer.push(`(last_activity < ${a} OR (last_activity = ${b} AND session_id < ${sid}))`);
-        }
-        const sql = "SELECT session_id, owner, task_id, last_activity, first_activity, run_count, objective_preview, status, last_task_run_id FROM (" +
-            "SELECT session_id, owner, task_id, objective_preview, status, result ->> 'runId' AS last_task_run_id," +
-            " ROW_NUMBER() OVER (PARTITION BY session_id ORDER BY created_at DESC, task_id DESC) AS rn," +
-            " MAX(created_at) OVER (PARTITION BY session_id) AS last_activity," +
-            " MIN(created_at) OVER (PARTITION BY session_id) AS first_activity," +
-            " COUNT(*) OVER (PARTITION BY session_id) AS run_count" +
-            " FROM task_run" + innerWhere + ") t WHERE " + outer.join(" AND ") +
-            ` ORDER BY last_activity DESC, session_id DESC LIMIT ${p(opts.limit)}`;
-        const { rows } = await this.db.query(sql, params);
-        return rows.map((r) => this.rowToSession(r));
-    }
-    rowToSession(r) {
-        const lastTaskRunId = r.last_task_run_id;
-        return {
-            sessionId: String(r.session_id),
-            owner: r.owner ?? null,
-            lastActivityAt: iso(r.last_activity),
-            firstActivityAt: iso(r.first_activity),
-            runCount: Number(r.run_count),
-            objectivePreview: r.objective_preview ?? null,
-            lastStatus: String(r.status),
-            lastRunId: r.task_id ?? null,
-            ...(typeof lastTaskRunId === "string" && lastTaskRunId !== "" ? { lastTaskRunId } : {}),
-            title: null,
-            pinned: false,
-        };
-    }
-    async deleteBySession(sessionId, expectedSessionOwner) {
+    async deleteBySession(sessionId) {
         let result = { removed: 0 };
         await this.tx(async (conn) => {
-            const { rows: meta } = await conn.query(this.q("SELECT owner FROM session_meta WHERE session_id = ? FOR UPDATE", "SELECT owner FROM session_meta WHERE session_id = $1 FOR UPDATE"), [sessionId]);
-            const metaRow = meta[0];
-            if (metaRow === undefined || (metaRow.owner ?? null) !== expectedSessionOwner)
-                return;
-            const { rows: active } = await conn.query(this.q("SELECT task_id FROM task_active WHERE session_id = ? FOR UPDATE", "SELECT task_id FROM task_active WHERE session_id = $1 FOR UPDATE"), [sessionId]);
-            if (active[0]) {
-                result = { active: String(active[0].task_id) };
-                return;
-            }
             await conn.query(this.q("DELETE te FROM task_event te JOIN task_run tr ON te.task_id = tr.task_id WHERE tr.session_id = ?", "DELETE FROM task_event te USING task_run tr WHERE te.task_id = tr.task_id AND tr.session_id = $1"), [sessionId]);
             await conn.query(this.q("DELETE ta FROM task_active ta JOIN task_run tr ON ta.task_id = tr.task_id WHERE tr.session_id = ?", "DELETE FROM task_active ta USING task_run tr WHERE ta.task_id = tr.task_id AND tr.session_id = $1"), [sessionId]);
             const { affected } = await conn.query(this.q("DELETE FROM task_run WHERE session_id = ?", "DELETE FROM task_run WHERE session_id = $1"), [sessionId]);
@@ -553,17 +595,14 @@ export class SqlRunStore {
             ts: iso(r.ts),
         }));
     }
-    async reapStale(olderThanMs) {
-        const cutoff = new Date(Date.now() - olderThanMs);
+    async reapStale() {
+        const cutoff = new Date(Date.now() - this.stall.windowMs());
         let reaped = 0;
         await this.tx(async (conn) => {
-            const { affected } = await conn.query(this.q(`UPDATE task_run SET status = 'failed', result = NULL, error = '${RUN_STALLED_ERROR}', error_code = '${RUN_HOLDER_DEATH_CODES.stalled}', updated_at = ? ` +
-                "WHERE status = 'running' AND updated_at < ?", `UPDATE task_run SET status = 'failed', result = NULL, error = '${RUN_STALLED_ERROR}', error_code = '${RUN_HOLDER_DEATH_CODES.stalled}', updated_at = $1 ` +
-                "WHERE status = 'running' AND updated_at < $2"), [new Date(), cutoff]);
-            reaped = affected;
-            await conn.query(this.q("DELETE ta FROM task_active ta JOIN task_run tr ON ta.task_id = tr.task_id WHERE tr.status NOT IN ('running','suspended','needs_review')", "DELETE FROM task_active ta USING task_run tr WHERE ta.task_id = tr.task_id AND tr.status NOT IN ('running','suspended','needs_review')"));
-            await conn.query(this.q("DELETE FROM task_active WHERE NOT EXISTS (SELECT 1 FROM task_run tr WHERE tr.task_id = task_active.task_id)", "DELETE FROM task_active WHERE NOT EXISTS (SELECT 1 FROM task_run tr WHERE tr.task_id = task_active.task_id)"));
+            reaped = await this.landStallVerdicts(conn, cutoff);
         });
+        if (reaped > 0)
+            this.stall.onVerdicts(reaped);
         return reaped;
     }
     async reapSuspended(olderThanMs) {
@@ -572,9 +611,9 @@ export class SqlRunStore {
         await this.tx(async (conn) => {
             const { affected } = await conn.query(this.q("UPDATE task_run tr SET tr.status = 'failed', tr.result = NULL, tr.error = 'approval expired before decision', tr.error_code = 'approval.expired', tr.updated_at = ? " +
                 "WHERE tr.status = 'suspended' AND tr.updated_at < ? " +
-                "AND NOT EXISTS (SELECT 1 FROM checkpoint c WHERE c.session_id = tr.session_id AND c.status = 'pending')", "UPDATE task_run AS tr SET status = 'failed', result = NULL, error = 'approval expired before decision', error_code = 'approval.expired', updated_at = $1 " +
+                "AND NOT EXISTS (SELECT 1 FROM checkpoint c WHERE c.host_session_id = tr.session_id AND c.status = 'pending')", "UPDATE task_run AS tr SET status = 'failed', result = NULL, error = 'approval expired before decision', error_code = 'approval.expired', updated_at = $1 " +
                 "WHERE tr.status = 'suspended' AND tr.updated_at < $2 " +
-                "AND NOT EXISTS (SELECT 1 FROM checkpoint c WHERE c.session_id = tr.session_id AND c.status = 'pending')"), [new Date(), cutoff]);
+                "AND NOT EXISTS (SELECT 1 FROM checkpoint c WHERE c.host_session_id = tr.session_id AND c.status = 'pending')"), [new Date(), cutoff]);
             reaped = affected;
             await conn.query(this.q("DELETE ta FROM task_active ta JOIN task_run tr ON ta.task_id = tr.task_id WHERE tr.status NOT IN ('running','suspended','needs_review')", "DELETE FROM task_active ta USING task_run tr WHERE ta.task_id = tr.task_id AND tr.status NOT IN ('running','suspended','needs_review')"));
         });
@@ -585,11 +624,11 @@ export class SqlRunStore {
         await this.tx(async (conn) => {
             const { affected } = await conn.query(this.q("UPDATE task_run tr SET tr.status = 'failed', tr.result = NULL, tr.error = 'approval expired before decision', tr.error_code = 'approval.expired', tr.updated_at = ? " +
                 "WHERE tr.status IN ('suspended','needs_review') " +
-                "AND EXISTS (SELECT 1 FROM checkpoint ce WHERE ce.session_id = tr.session_id AND ce.status = 'expired') " +
-                "AND NOT EXISTS (SELECT 1 FROM checkpoint cp WHERE cp.session_id = tr.session_id AND cp.status = 'pending')", "UPDATE task_run AS tr SET status = 'failed', result = NULL, error = 'approval expired before decision', error_code = 'approval.expired', updated_at = $1 " +
+                "AND EXISTS (SELECT 1 FROM checkpoint ce WHERE ce.host_session_id = tr.session_id AND ce.status = 'expired') " +
+                "AND NOT EXISTS (SELECT 1 FROM checkpoint cp WHERE cp.host_session_id = tr.session_id AND cp.status = 'pending')", "UPDATE task_run AS tr SET status = 'failed', result = NULL, error = 'approval expired before decision', error_code = 'approval.expired', updated_at = $1 " +
                 "WHERE tr.status IN ('suspended','needs_review') " +
-                "AND EXISTS (SELECT 1 FROM checkpoint ce WHERE ce.session_id = tr.session_id AND ce.status = 'expired') " +
-                "AND NOT EXISTS (SELECT 1 FROM checkpoint cp WHERE cp.session_id = tr.session_id AND cp.status = 'pending')"), [new Date()]);
+                "AND EXISTS (SELECT 1 FROM checkpoint ce WHERE ce.host_session_id = tr.session_id AND ce.status = 'expired') " +
+                "AND NOT EXISTS (SELECT 1 FROM checkpoint cp WHERE cp.host_session_id = tr.session_id AND cp.status = 'pending')"), [new Date()]);
             reaped = affected;
             await conn.query(this.q("DELETE ta FROM task_active ta JOIN task_run tr ON ta.task_id = tr.task_id WHERE tr.status NOT IN ('running','suspended','needs_review')", "DELETE FROM task_active ta USING task_run tr WHERE ta.task_id = tr.task_id AND tr.status NOT IN ('running','suspended','needs_review')"));
         });
@@ -597,13 +636,13 @@ export class SqlRunStore {
     }
 }
 export class TiDBRunStore extends SqlRunStore {
-    constructor(pool) {
-        super(mysqlDriver(pool));
+    constructor(pool, stall) {
+        super(mysqlDriver(pool), stall);
     }
 }
 export class PgRunStore extends SqlRunStore {
-    constructor(pool) {
-        super(pgDriver(pool));
+    constructor(pool, stall) {
+        super(pgDriver(pool), stall);
     }
 }
 //# sourceMappingURL=run-store-sql.js.map

package/dist/plugins/s3-answer.d.ts

@@ -6,7 +6,8 @@
  * the second as the first turns a missing bucket into a silently empty workspace on every run. The ONLY evidence of a missing bucket is
  * a 404 whose body carries the S3 error code `NoSuchBucket`: a bare 404, a 404 from a proxy or a wrong path, a 403 (credentials that
  * cannot list ≠ no bucket) and a 5xx prove nothing about the bucket. Every consumer asks this one predicate: the boot probe of a
- * configured bucket (`MinioBlobBackend.probeBucket`) and the runtime read of the workspace `latest` pointer (`remote-env-k8s.ts`).
+ * configured bucket (`MinioBlobBackend.probeBucket`), the runtime read of the workspace `latest` pointer, and a restore's archive GET whose 404 answer the
+ * pod hands back as it is (`remote-env-k8s.ts` — the pod never judges it).
  */
 /** The code a run's refusal / operator line carries when the store answered "no such bucket" (the `workspace_snapshot_bucket_missing` line's `code`). */
 export declare const SNAPSHOT_BUCKET_MISSING = "snapshot_bucket_missing";

package/dist/plugins/sandbox-reaper.d.ts

@@ -0,0 +1,26 @@
+/**
+ * 车道沙箱回收腿的接口 —— 维护 tick(`boot/reapers.ts`)上**唯一一只**沙箱回收座:k8s 车道 = 沙箱 Pod 回收器({@link createSandboxPodReaper},`k8s-pod-reaper.ts`),
+ * local-docker 车道 = 无主容器回收器({@link createLocalDockerReaper},`local-docker-reaper.ts`)。一次部署只有一条执行车道,所以座只有一只、节拍只有一个
+ * ({@link SANDBOX_REAP_PERIOD_MS})、连败告警只有一个名;两条车道各自的判据与读数留在各自的模块里,这里只有 tick 要知道的那一点。
+ */
+/** 一个副本两轮沙箱回收之间的间隔。常量:起服后第一个维护 tick 即跑一轮(起服那一轮),此后每个周期一轮。 */
+export declare const SANDBOX_REAP_PERIOD_MS: number;
+/** run 店的窄读口(三只 run 店共有的 `getRun`):回收器按 run 行的终态判「这只沙箱还有没有主人」。 */
+export interface SandboxReaperRunReader {
+    getRun(taskId: string): Promise<{
+        readonly status: string;
+        readonly updatedAt: string;
+    } | null | undefined>;
+}
+/** 一轮的公共读数(两条车道的轮次读数都是它的超集):列出来了没有 / 扫了几只 / 按各自闭集原因分的删与跳过计数。 */
+export interface SandboxReapRound {
+    readonly listed: boolean;
+    readonly scanned: number;
+    readonly deleted: Readonly<Record<string, number | undefined>>;
+    readonly skipped: Readonly<Record<string, number | undefined>>;
+}
+/** 维护 tick 调的那一口:一轮 = 列举 → 逐只判 → 收被判无主的;不抛。`runs` 缺席 = 本部署没有 run 店(判不了 run 的沙箱一律不动)。 */
+export interface SandboxReaper {
+    reapOnce(runs: SandboxReaperRunReader | undefined): Promise<SandboxReapRound>;
+}
+//# sourceMappingURL=sandbox-reaper.d.ts.map

package/dist/plugins/sandbox-reaper.js

@@ -0,0 +1,2 @@
+export const SANDBOX_REAP_PERIOD_MS = 5 * 60_000;
+//# sourceMappingURL=sandbox-reaper.js.map

package/dist/plugins/scenario-prefix-switch-m42.d.ts

@@ -0,0 +1,56 @@
+import { StoreBootRefusalError } from "./store-contracts.js";
+/** 本条的拒启码(机读)。 */
+export declare const SCENARIO_PREFIX_SWITCH_REFUSED_CODE = "config.scenario_prefix_switch_refused";
+/** 本条在标记表(`one-time-migration-marker.ts`)里的那一行的键。 */
+export declare const SCENARIO_PREFIX_MARKER_ID = "M42";
+/** local 车道的标记文件(数据根下,文件名即本条)。 */
+export declare const LOCAL_MARKER_FILE = "one-time-migration-M42.json";
+/** 身份列的名字判据(建表句派生扫描面的唯一一条)。 */
+export declare const IDENTITY_COLUMN_NAME: RegExp;
+/** `LIKE` 的前缀模式(`!` 作转义符:两方言同义、不碰反斜杠的方言差;前缀里的 `_` 是通配符,必须转义)。 */
+export declare const SCENARIO_PREFIX_LIKE: string;
+/** 一处命中:来源(SQL = `表.列`;local = 店名)+ 以保留前缀开头的行数。 */
+export interface ScenarioPrefixHit {
+    readonly source: string;
+    readonly rows: number;
+}
+/** 本条的读数 / 收据格(扫了几处来源;0 = 标记在场、没扫)。 */
+export interface ScenarioPrefixReceipt {
+    scenarioPrefixSourcesScanned: number;
+}
+/** 拒因闭集:存量命中 / 零命中而更早版本的副本可能仍在服务(SQL 的停-迁-起)。 */
+export type ScenarioPrefixRefusalReason = "stored_prefix" | "live_runs";
+/** 判活读数(`live_runs` 那一拒带):判活窗内心跳过的 run 数与可以再起的最早时刻。 */
+export interface ScenarioPrefixLiveRuns {
+    readonly liveRuns: number;
+    readonly windowSec: number;
+    readonly restartAfter: string;
+    readonly after: string;
+}
+/**
+ * M42 的起服拒。具名类型 + 机读码 + 拒因;继承 {@link StoreBootRefusalError}(SQL 开库的等待窗不重试它;隐式 local 的降级臂不吞)。一个字节没改(扫描只读,标记没写)。
+ *  · `stored_prefix` —— 存量里有以保留前缀开头的身份(`hits` 逐来源点名);
+ *  · `live_runs` —— 零命中,但判活窗内仍有 run 在心跳:更早版本的副本可能还在服务,扫描之后它还会收下这类身份 ⇒ 全停之后再起(`live` 带时刻)。
+ */
+export declare class ScenarioPrefixSwitchRefusedError extends StoreBootRefusalError {
+    readonly reason: ScenarioPrefixRefusalReason;
+    readonly hits: readonly ScenarioPrefixHit[];
+    readonly live?: ScenarioPrefixLiveRuns | undefined;
+    readonly code = "config.scenario_prefix_switch_refused";
+    constructor(reason: ScenarioPrefixRefusalReason, hits: readonly ScenarioPrefixHit[], live?: ScenarioPrefixLiveRuns | undefined, 
+    /** 库此刻还差的列(SQL 腿经 `describeUpgradeDriftTail` 读到的;列齐 / local 腿 ⇒ 空串):一次起服读全。 */
+    tail?: string);
+    private static sentence;
+}
+/** local 腿的一处来源:店名 + 它的计数口(以保留前缀开头的属主行数)。 */
+export interface LocalScenarioPrefixSource {
+    readonly source: string;
+    readonly rows: () => number | Promise<number>;
+}
+/**
+ * 🔴 M42 的 local 腿(唯一调用点 `LocalBackend.runOneTimeMigrations`,第一步):标记文件在 ⇒ 0(零扫描);否则逐店计数,一处命中 ⇒ {@link ScenarioPrefixSwitchRefusedError}
+ * (一字节没写);零命中 ⇒ 原子写标记文件(`required`:标记没落盘就不算扫过 —— 写不进 ⇒ 抛,起服拒,下一次起服重扫)。返回扫了几处来源。
+ * 数据根独占(根锁):同一只根上没有第二个起服者、也没有更早版本的副本与本次并跑 ⇒ 没有 SQL 腿的判活守卫。
+ */
+export declare function switchLocalScenarioPrefixM42(markerPath: string, sources: readonly LocalScenarioPrefixSource[]): Promise<number>;
+//# sourceMappingURL=scenario-prefix-switch-m42.d.ts.map

package/dist/plugins/scenario-prefix-switch-m42.js

@@ -0,0 +1,56 @@
+import { existsSync } from "node:fs";
+import { SCENARIO_SUBJECT_PREFIX } from "../owner-brands.js";
+import { atomicWriteFileSync } from "./owner-only-file-mode.js";
+import { StoreBootRefusalError } from "./store-contracts.js";
+import { stopThenStartAction } from "./stop-then-start-sentence.js";
+export const SCENARIO_PREFIX_SWITCH_REFUSED_CODE = "config.scenario_prefix_switch_refused";
+export const SCENARIO_PREFIX_MARKER_ID = "M42";
+export const LOCAL_MARKER_FILE = "one-time-migration-M42.json";
+export const IDENTITY_COLUMN_NAME = /(^|_)(owner|scope|principal|subject)$/;
+export const SCENARIO_PREFIX_LIKE = `${SCENARIO_SUBJECT_PREFIX.replace(/[!%_]/g, "!$&")}%`;
+export class ScenarioPrefixSwitchRefusedError extends StoreBootRefusalError {
+    reason;
+    hits;
+    live;
+    code = SCENARIO_PREFIX_SWITCH_REFUSED_CODE;
+    constructor(reason, hits, live, tail = "") {
+        super(ScenarioPrefixSwitchRefusedError.sentence(reason, hits, live) + tail);
+        this.reason = reason;
+        this.hits = hits;
+        this.live = live;
+        this.name = "ScenarioPrefixSwitchRefusedError";
+    }
+    static sentence(reason, hits, live) {
+        switch (reason) {
+            case "stored_prefix":
+                return (`${SCENARIO_PREFIX_SWITCH_REFUSED_CODE}: stored identities start with the reserved prefix "${SCENARIO_SUBJECT_PREFIX}" (` +
+                    hits.map((h) => `${h.source}: ${h.rows} row${h.rows === 1 ? "" : "s"}`).join(", ") +
+                    `). This version reads that prefix as a scenario of a person, so these rows would be attributed to another person; it refuses to start rather than misread them. ` +
+                    `Nothing was changed. Rename or delete those identities (the identity boundary now rejects principals with this prefix), then restart — ` +
+                    `the check runs until it finds none, then records a marker in the store and does not run again.`);
+            case "live_runs":
+                return (`${SCENARIO_PREFIX_SWITCH_REFUSED_CODE}: no stored identity starts with the reserved prefix "${SCENARIO_SUBJECT_PREFIX}", but ${live?.liveRuns ?? 0} run(s) heartbeated on this database ` +
+                    `within the last ${live?.windowSec ?? 0}s — a replica of an earlier version may still be serving, and it accepts principals with that prefix, so a clean check now would not stay clean. ` +
+                    `Nothing was changed and no marker was written. ${stopThenStartAction(live?.after)}`);
+            default: {
+                const unhandled = reason;
+                throw new Error(`scenario-prefix-switch: unknown refusal reason ${String(unhandled)}`);
+            }
+        }
+    }
+}
+export async function switchLocalScenarioPrefixM42(markerPath, sources) {
+    if (existsSync(markerPath))
+        return 0;
+    const hits = [];
+    for (const s of sources) {
+        const rows = await s.rows();
+        if (rows > 0)
+            hits.push({ source: s.source, rows });
+    }
+    if (hits.length > 0)
+        throw new ScenarioPrefixSwitchRefusedError("stored_prefix", hits);
+    atomicWriteFileSync(markerPath, JSON.stringify({ migration: SCENARIO_PREFIX_MARKER_ID, markedAt: new Date().toISOString() }), { durability: "required" });
+    return sources.length;
+}
+//# sourceMappingURL=scenario-prefix-switch-m42.js.map