@desplega.ai/agent-swarm 2.7.5 → 2.7.6

package/src/be/db.ts

@@ -313,7 +313,11 @@ export {
   getContextVersionHistory,
   getLatestContextVersion,
 } from "./db/context-versions";
-export { getScriptRunExecutionArgs } from "./db/script-runs";
+export {
+  getScriptRunExecutionArgs,
+  getScriptRunFollowUpGuard,
+  setScriptRunFollowUpGuard,
+} from "./db/script-runs";
 export {
   AssetKeyMovesBlockedError,
   assertAssetKeyMovesAllowed,
@@ -336,12 +340,14 @@ export {
 } from "./db/runtime";
 
 export {
+  countAgentDispatchesInThread,
   type ExistingTrackerContextWork,
   type ExistingTrackerContextWorkReason,
   findActiveTasksByVcs,
-  findCompletedTaskInThread,
+  findCompletedTasksInThread,
   findExistingLinearTrackerContextWork,
   findRecentCancelledTaskInThread,
+  findRecentFailedTaskInThread,
   findTaskByGitHub,
   findTaskByVcs,
   getAgentCurrentTask,
@@ -14116,6 +14122,7 @@ type ScriptRunRow = {
   last_heartbeat_at: string | null;
   idempotencyKey: string | null;
   requestedByUserId: string | null;
+  sourceTaskId: string | null;
   created_by: string | null;
   updated_by: string | null;
 };
@@ -14160,6 +14167,7 @@ function rowToScriptRun(row: ScriptRunRow): ScriptRun {
     lastHeartbeatAt: row.last_heartbeat_at ?? undefined,
     idempotencyKey: row.idempotencyKey ?? undefined,
     requestedByUserId: row.requestedByUserId ?? undefined,
+    sourceTaskId: row.sourceTaskId ?? undefined,
   };
 }
 
@@ -14188,6 +14196,7 @@ export async function createScriptRun(data: {
   scriptName?: string;
   idempotencyKey?: string;
   requestedByUserId?: string;
+  sourceTaskId?: string;
   createdBy?: string;
   updatedBy?: string;
 }): Promise<{ run: ScriptRun; existing: boolean }> {
@@ -14202,8 +14211,9 @@ export async function createScriptRun(data: {
 
   const row = await client.get<ScriptRunRow>(
     `INSERT INTO script_runs
-        (id, agentId, scriptName, source, args, idempotencyKey, requestedByUserId, created_by, updated_by)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        (id, agentId, scriptName, source, args, idempotencyKey, requestedByUserId, sourceTaskId,
+         created_by, updated_by)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        RETURNING *`,
     [
       data.id,
@@ -14216,6 +14226,7 @@ export async function createScriptRun(data: {
       sealJson(data.args ?? null),
       data.idempotencyKey ?? null,
       data.requestedByUserId ?? null,
+      data.sourceTaskId ?? null,
       data.createdBy ?? null,
       data.updatedBy ?? data.createdBy ?? null,
     ],

package/src/be/memory/providers/sqlite-store.ts

@@ -45,14 +45,18 @@ function armFetchLimit(limit: number): number {
 // lookup against it is a full table scan per outer row. The old backfill used
 // `WHERE NOT EXISTS (SELECT 1 FROM memory_fts f WHERE f.memory_id = m.id)`,
 // which is N x N: at ~18k memories it held the main connection for minutes at
-// boot. Both statements below use a non-correlated `NOT IN (subquery)`, which
-// SQLite evaluates once into an ephemeral index, so each is one pass over each
-// table. `memory_id IS NOT NULL` keeps a stray NULL row from turning every
-// NOT IN into NULL (which would insert nothing).
-export const FTS_DELETE_EXTRA_SQL = `DELETE FROM memory_fts
-  WHERE memory_id NOT IN (SELECT id FROM agent_memory)`;
-export const FTS_MISSING_IDS_SQL = `SELECT m.id FROM agent_memory m
-  WHERE m.id NOT IN (SELECT memory_id FROM memory_fts WHERE memory_id IS NOT NULL)`;
+// boot. The diff below reads each table once, in rowid batches (FTS5 seeks a
+// rowid range), and yields the event loop between batches: one whole-table
+// statement held the loop for seconds on a large DB. A memory_fts row is an
+// orphan when its memory is gone; that check is a lookup on the agent_memory
+// primary key. A NULL memory_id is never an orphan, as with the old NOT IN.
+export const FTS_SCAN_BATCH_SQL = `SELECT rowid AS rowid, memory_id AS memoryId,
+  memory_id IS NOT NULL
+    AND NOT EXISTS (SELECT 1 FROM agent_memory m WHERE m.id = memory_fts.memory_id) AS orphan
+  FROM memory_fts WHERE rowid > ? ORDER BY rowid LIMIT ?`;
+export const MEMORY_ID_BATCH_SQL = `SELECT rowid AS rowid, id FROM agent_memory
+  WHERE rowid > ? ORDER BY rowid LIMIT ?`;
+const FTS_SCAN_BATCH_SIZE = 1000;
 const FTS_POPULATE_BATCH_SIZE = 500;
 
 export type AgentMemoryRow = {
@@ -251,12 +255,45 @@ export class SqliteMemoryStore implements MemoryStore {
   private async populateFtsTable(): Promise<void> {
     const client = getDbClient();
     const startedAt = performance.now();
-    const deletedExtra = await client.run(FTS_DELETE_EXTRA_SQL);
-    if (deletedExtra.changes > 0) {
-      console.warn(`[memory-fts] removed_extra_rows count=${deletedExtra.changes}`);
+    const yieldLoop = () => new Promise<void>((resolve) => setImmediate(resolve));
+
+    // Drop orphan rows and collect the memory ids the index holds.
+    const indexed = new Set<string>();
+    let removed = 0;
+    for (let after = 0; ; ) {
+      const rows = await client.query<{ rowid: number; memoryId: string | null; orphan: number }>(
+        FTS_SCAN_BATCH_SQL,
+        [after, FTS_SCAN_BATCH_SIZE],
+      );
+      if (rows.length === 0) break;
+      after = rows[rows.length - 1]!.rowid;
+      const orphans = rows.filter((row) => row.orphan).map((row) => row.rowid);
+      if (orphans.length > 0) {
+        await client.run(
+          `DELETE FROM memory_fts WHERE rowid IN (${orphans.map(() => "?").join(",")})`,
+          orphans,
+        );
+        removed += orphans.length;
+      }
+      for (const row of rows) if (!row.orphan && row.memoryId) indexed.add(row.memoryId);
+      await yieldLoop();
+    }
+    if (removed > 0) {
+      console.warn(`[memory-fts] removed_extra_rows count=${removed}`);
+    }
+
+    const missing: { id: string }[] = [];
+    for (let after = 0; ; ) {
+      const rows = await client.query<{ rowid: number; id: string }>(MEMORY_ID_BATCH_SQL, [
+        after,
+        FTS_SCAN_BATCH_SIZE,
+      ]);
+      if (rows.length === 0) break;
+      after = rows[rows.length - 1]!.rowid;
+      for (const row of rows) if (!indexed.has(row.id)) missing.push(row);
+      await yieldLoop();
     }
 
-    const missing = await client.query<{ id: string }>(FTS_MISSING_IDS_SQL);
     for (let i = 0; i < missing.length; i += FTS_POPULATE_BATCH_SIZE) {
       const ids = missing.slice(i, i + FTS_POPULATE_BATCH_SIZE).map((row) => row.id);
       const placeholders = ids.map(() => "?").join(",");

package/src/be/migrations/260_runtime_pool_cache.sql

@@ -0,0 +1,20 @@
+-- NULL preserves the ephemeral-only behavior of existing pools.
+ALTER TABLE runtime_pools ADD COLUMN cache TEXT
+  CHECK (cache IS NULL OR json_valid(cache));
+
+-- Retain release intent across API restarts and pool deletion.
+CREATE TABLE runtime_cache_releases (
+  id TEXT PRIMARY KEY,
+  pool_id TEXT NOT NULL,
+  controller_id TEXT NOT NULL REFERENCES runtime_controllers(id) ON DELETE RESTRICT,
+  owner_id TEXT NOT NULL,
+  cache_key TEXT NOT NULL,
+  runtime_ids TEXT NOT NULL CHECK (json_valid(runtime_ids) AND json_type(runtime_ids) = 'array'),
+  state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'deleting')),
+  last_error TEXT,
+  created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+  created_by TEXT REFERENCES users(id) ON DELETE SET NULL,
+  updated_by TEXT REFERENCES users(id) ON DELETE SET NULL,
+  UNIQUE(controller_id, owner_id, cache_key)
+);

package/src/be/migrations/262_session_visibility_indexes.sql

@@ -0,0 +1,18 @@
+-- Covering indexes for the session visibility walk (readableSessionCondition
+-- in src/be/db/tasks/read.ts). Every task list and count a dashboard user
+-- loads computes the set of tasks in sessions the user cannot read: roots
+-- owned by someone else, plus all their descendants.
+--
+-- agent_tasks rows carry large `task` text, and parentTaskId, scheduleId and
+-- requestedByUserId sit after it, in the overflow pages. Without these
+-- indexes the root step read every unscheduled task row and the recursive
+-- step read the row of every descendant. With them, both steps stay in the
+-- index. Production has no sqlite_stat1; the planner picks both without it.
+
+-- Root step: `scheduleId IS NULL` plus the root, owner and id columns.
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_session_owner
+  ON agent_tasks(scheduleId, parentTaskId, requestedByUserId, id);
+
+-- Recursive step: the children of a hidden task, by id.
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_parentTaskId_id
+  ON agent_tasks(parentTaskId, id);

package/src/be/migrations/263_task_list_filter_index.sql

@@ -0,0 +1,11 @@
+-- Covering index for the default task list count (getTasksCount in
+-- src/be/db/tasks/read.ts). The task list hides heartbeat tasks by default:
+--
+--   IFNULL(taskType, '') NOT IN (...) AND tags NOT LIKE '%"heartbeat"%'
+--
+-- A NOT IN and a NOT LIKE cannot seek, so the count scans. taskType and tags
+-- sit after the large `task` text, in the overflow pages, so the scan read
+-- every task body. This index holds the two filter columns and `id` (the
+-- session visibility filter), so the count scans the index only.
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_list_filter
+  ON agent_tasks(taskType, tags, id);

package/src/be/migrations/264_slack_thread_cover_index.sql

@@ -0,0 +1,9 @@
+-- Covering index for the Slack render-v2 watcher sweep (getSlackTreeMessages
+-- in src/be/db.ts), which runs on every watcher tick, even when idle. For
+-- each stored tree it tests the thread's tasks by createdAt, status, source
+-- and lastUpdatedAt. Those columns sit after the large `task` text, in the
+-- overflow pages, so each test read whole task rows. This index holds them,
+-- so the tests stay in the index. Partial: only Slack tasks carry a channel.
+CREATE INDEX IF NOT EXISTS idx_agent_tasks_slack_thread_cover
+  ON agent_tasks(slackChannelId, slackThreadTs, createdAt, status, source, lastUpdatedAt, id)
+  WHERE slackChannelId IS NOT NULL;

package/src/be/migrations/266_script_runs_source_task.sql

@@ -0,0 +1,11 @@
+-- The task whose session launched a durable script run (the caller's
+-- X-Source-Task-Id, kept only when that task belongs to the launching agent).
+-- The follow-up re-delegation guard reads it when the run's ctx.step.agentTask
+-- dispatches, the same way send-task reads the caller's source task.
+ALTER TABLE script_runs ADD COLUMN sourceTaskId TEXT;
+
+-- The run's follow-up re-delegation guard decision, taken once for the whole
+-- run at its first new step: NULL (not decided yet), 'passed', or the rule that
+-- blocked it ('same-work' or 'loop-cap'). Every later step follows it, so
+-- parallel steps of one run are dispatched all or none.
+ALTER TABLE script_runs ADD COLUMN followUpGuard TEXT;

package/src/be/runtime-pools/cache-cleanup.ts

@@ -0,0 +1,109 @@
+import type { ControllerCommands } from "../../runtime-providers/controller";
+import { poolCacheKey, type RuntimeCacheReference } from "../../runtime-providers/types";
+import type { RuntimePool } from "../../types";
+import { scrubSecrets } from "../../utils/secret-scrubber";
+import { getDbClient, getSwarmConfigs } from "../db";
+
+/** The stored key is private API metadata; legacy caches retain their original name. */
+export async function getPoolCache(poolId: string): Promise<{
+  key: string;
+  mountPath: string;
+  controllerId: string;
+} | null> {
+  const row = await getDbClient().get<{ cache: string | null; controllerId: string }>(
+    "SELECT cache, json_extract(placements, '$[0]') AS controllerId FROM runtime_pools WHERE id = ?",
+    [poolId],
+  );
+  if (!row?.cache) return null;
+  const cache = JSON.parse(row.cache);
+  return {
+    key: cache.key ?? poolCacheKey(poolId),
+    mountPath: cache.mountPath,
+    controllerId: row.controllerId,
+  };
+}
+
+/** Call in the pool write transaction; no infrastructure calls occur here. */
+export async function queuePoolCacheRelease(
+  pool: RuntimePool,
+  actor: string | null,
+): Promise<void> {
+  if (!pool.cache) return;
+  const owner = (await getSwarmConfigs({ scope: "global", key: "runtime_owner_id" }))[0];
+  // No runtime could have created an owned cache before the owner was allocated.
+  if (!owner) return;
+  const cache = (await getPoolCache(pool.id))!;
+  const runtimes = await getDbClient().query<{ id: string }>(
+    `SELECT id FROM runtime_instances WHERE pool_id = ?
+     AND (controller_id = ? OR controller_id IS NULL) AND status != 'offline'`,
+    [pool.id, pool.placements[0]!],
+  );
+  await getDbClient().run(
+    `INSERT INTO runtime_cache_releases
+      (id, pool_id, controller_id, owner_id, cache_key, runtime_ids, created_by, updated_by)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(controller_id, owner_id, cache_key) DO NOTHING`,
+    [
+      crypto.randomUUID(),
+      pool.id,
+      pool.placements[0]!,
+      owner.value,
+      cache.key,
+      JSON.stringify(runtimes.map((r) => r.id)),
+      actor,
+      actor,
+    ],
+  );
+}
+
+interface CacheRelease extends RuntimeCacheReference {
+  id: string;
+  controllerId: string;
+}
+
+const RELEASE_LEASE_MS = 5 * 60_000;
+
+/** Claims are atomic across reconcilers; all network calls are outside DB transactions. */
+export async function reconcileCacheReleases(
+  commands: (id: string) => Promise<ControllerCommands>,
+  signal: AbortSignal,
+  now?: Date,
+): Promise<void> {
+  const jobs = await getDbClient().query<CacheRelease>(
+    `SELECT id, pool_id AS poolId, controller_id AS controllerId, owner_id AS ownerId,
+     cache_key AS key FROM runtime_cache_releases ORDER BY created_at`,
+  );
+  for (const job of jobs) {
+    if (signal.aborted) return;
+    const claimed = now ?? new Date();
+    const claimedAt = claimed.toISOString();
+    // Reclaim abandoned work after five minutes; generation keys fence late provider calls.
+    const reserved = await getDbClient().run(
+      `UPDATE runtime_cache_releases SET state = 'deleting', updated_at = ?
+       WHERE id = ? AND (state = 'pending' OR updated_at < ?) AND NOT EXISTS (
+         SELECT 1 FROM runtime_instances WHERE status != 'offline'
+         AND id IN (SELECT value FROM json_each(runtime_cache_releases.runtime_ids))
+       )`,
+      [claimedAt, job.id, new Date(claimed.getTime() - RELEASE_LEASE_MS).toISOString()],
+    );
+    if (!reserved.changes) continue;
+    try {
+      signal.throwIfAborted();
+      const controller = await commands(job.controllerId);
+      if (!controller.releaseCache)
+        throw new Error("Controller cannot release caches; upgrade it before cleanup");
+      await controller.releaseCache({ key: job.key, poolId: job.poolId, ownerId: job.ownerId });
+      await getDbClient().run(
+        "DELETE FROM runtime_cache_releases WHERE id = ? AND state = 'deleting' AND updated_at = ?",
+        [job.id, claimedAt],
+      );
+    } catch (error) {
+      const message = scrubSecrets(String(error));
+      await getDbClient().run(
+        `UPDATE runtime_cache_releases SET state = 'pending', last_error = ?, updated_at = ?
+         WHERE id = ? AND state = 'deleting' AND updated_at = ?`,
+        [message, new Date().toISOString(), job.id, claimedAt],
+      );
+      if (!signal.aborted) console.error("[runtime-pools] Cache cleanup deferred:", message);
+    }
+  }
+}

package/src/be/runtime-pools/db.ts

@@ -4,6 +4,7 @@ import {
   createRuntimeProvider,
   type RuntimeTargetConfig,
 } from "../../runtime-providers";
+import { poolCacheKey } from "../../runtime-providers/types";
 import type { Agent, DeliveryMode, RuntimeController, RuntimePool } from "../../types";
 import {
   RuntimeControllerCreateSchema,
@@ -17,6 +18,7 @@ import { decryptSecret, encryptSecret, getEncryptionKey } from "../crypto";
 import { createAgent, getAgentById, getAllAgents, getDbClient, upsertSwarmConfig } from "../db";
 import { type AgentRow, rowToAgent } from "../db/agents";
 import { registerStoredSecret } from "../secret-registry";
+import { getPoolCache, queuePoolCacheRelease } from "./cache-cleanup";
 import { RuntimePoolError } from "./errors";
 import { revokeControllerTokens } from "./tokens";
 
@@ -29,10 +31,11 @@ type ControllerRow = Omit<
   config: string;
   secretConfig: string | null;
 };
-type PoolRow = Omit<RuntimePool, "placements" | "env" | "resources" | "enabled"> & {
+type PoolRow = Omit<RuntimePool, "placements" | "env" | "resources" | "cache" | "enabled"> & {
   placements: string;
   env: string;
   resources: string | null;
+  cache: string | null;
   enabled: number;
 };
 const controllerColumns = `id, name, kind, placement, config, secret_config AS secretConfig,
@@ -40,7 +43,7 @@ const controllerColumns = `id, name, kind, placement, config, secret_config AS s
   created_by AS createdBy, updated_by AS updatedBy, created_at AS createdAt, updated_at AS updatedAt`;
 const poolColumns = `id, name, kind, placements, image, lifecycle, idle_minutes AS idleMinutes,
   slots_per_runtime AS slotsPerRuntime, max_runtimes AS maxRuntimes, min_runtimes AS minRuntimes,
-  delivery, env, resources, ttl_sec AS ttlSec, resolved_image AS resolvedImage, enabled,
+  delivery, env, resources, cache, ttl_sec AS ttlSec, resolved_image AS resolvedImage, enabled,
   consecutive_failures AS consecutiveFailures, circuit_open_until AS circuitOpenUntil,
   last_error AS lastError, last_error_at AS lastErrorAt, created_by AS createdBy,
   updated_by AS updatedBy, created_at AS createdAt, updated_at AS updatedAt`;
@@ -65,11 +68,13 @@ function rowToController(row: ControllerRow): RuntimeController {
   };
 }
 function rowToPool(row: PoolRow): RuntimePool {
+  const cache = row.cache ? JSON.parse(row.cache) : null;
   return {
     ...row,
     placements: JSON.parse(row.placements),
     env: JSON.parse(row.env),
     resources: row.resources ? JSON.parse(row.resources) : null,
+    cache: cache ? { mountPath: cache.mountPath } : null,
     enabled: row.enabled === 1,
   };
 }
@@ -121,6 +126,17 @@ export async function getRuntimeControllerTarget(
     E2B_MAX_CONTINUOUS_SEC: env.E2B_MAX_CONTINUOUS_SEC,
     RUNTIME_WORKER_API_URL: env.RUNTIME_WORKER_API_URL,
     PUBLIC_MCP_BASE_URL: env.PUBLIC_MCP_BASE_URL,
+    ...(controller.kind === "boxd" && controller.placement === "builtin"
+      ? {
+          BOXD_ORG: env.BOXD_ORG,
+          BOXD_BASE_URL: env.BOXD_BASE_URL,
+          BOXD_SNAPSHOT: env.BOXD_SNAPSHOT,
+          BOXD_DELETE_AFTER_SEC: env.BOXD_DELETE_AFTER_SEC,
+          BOXD_START_TIMEOUT_SEC: env.BOXD_START_TIMEOUT_SEC,
+          BOXD_STATUS_TIMEOUT_SEC: env.BOXD_STATUS_TIMEOUT_SEC,
+          BOXD_TEARDOWN_TIMEOUT_SEC: env.BOXD_TEARDOWN_TIMEOUT_SEC,
+        }
+      : {}),
     ...controller.config,
     ...secrets,
   };
@@ -234,11 +250,32 @@ export async function updateRuntimeController(
     )
       throw new RuntimePoolError("Controller name already exists", 409);
     const pools = await listRuntimePools();
+    const oldConfig: Record<string, string> = JSON.parse(row.config);
+    const configChanged =
+      Object.keys(data.config).length !== Object.keys(oldConfig).length ||
+      Object.entries(data.config).some(([key, value]) => oldConfig[key] !== value);
+    const secrets = patch.secretConfig !== undefined ? controllerSecrets(row) : {};
+    const secretsChanged = Object.entries(patch.secretConfig ?? {}).some(([key, value]) =>
+      value === null ? key in secrets : secrets[key] !== value,
+    );
+    if (
+      (data.kind !== row.kind ||
+        data.placement !== row.placement ||
+        configChanged ||
+        secretsChanged) &&
+      (pools.some((pool) => pool.cache && pool.placements.includes(id)) ||
+        (await tx.get("SELECT id FROM runtime_cache_releases WHERE controller_id = ? LIMIT 1", [
+          id,
+        ])))
+    )
+      throw new RuntimePoolError(
+        "Disable pool caches and wait for cleanup before changing controller configuration or placement",
+        409,
+      );
     if (pools.some((pool) => pool.placements.includes(id) && pool.kind !== data.kind))
       throw new RuntimePoolError("Pool kind must equal controller kind");
     let encrypted = row.secretConfig;
     if (patch.secretConfig !== undefined) {
-      const secrets = controllerSecrets(row);
       for (const [key, value] of Object.entries(patch.secretConfig)) {
         if (value === null) delete secrets[key];
         else secrets[key] = value;
@@ -272,6 +309,8 @@ export async function deleteRuntimeController(id: string): Promise<void> {
     if (!(await getRuntimeController(id))) throw new RuntimePoolError("Controller not found", 404);
     if ((await listRuntimePools()).some((pool) => pool.placements.includes(id)))
       throw new RuntimePoolError("Controller has runtime pools", 409);
+    if (await tx.get("SELECT id FROM runtime_cache_releases WHERE controller_id = ? LIMIT 1", [id]))
+      throw new RuntimePoolError("Controller has pending cache cleanup", 409);
     await revokeControllerTokens(id);
     await tx.run("DELETE FROM runtime_controllers WHERE id = ?", [id]);
   });
@@ -291,6 +330,9 @@ export async function getRuntimePool(id: string): Promise<RuntimePool | null> {
 }
 async function validatePool(data: z.output<typeof RuntimePoolCreateSchema>): Promise<void> {
   supportedKind(data.kind);
+  if (data.cache && !(await createRuntimeProvider(data.kind)).traits.persistentCache) {
+    throw new RuntimePoolError(`${data.kind} does not support persistent pool caches`);
+  }
   const controller = await getRuntimeController(data.placements[0]!);
   if (!controller) throw new RuntimePoolError("Controller not found");
   if (data.kind !== controller.kind)
@@ -316,7 +358,7 @@ export async function createRuntimePool(
     if (await tx.get("SELECT id FROM runtime_pools WHERE name = ? OR id = ?", [data.name, id]))
       throw new RuntimePoolError("Pool already exists", 409);
     await tx.run(
-      `INSERT INTO runtime_pools (id, name, kind, placements, image, lifecycle, idle_minutes, slots_per_runtime, max_runtimes, min_runtimes, delivery, env, resources, ttl_sec, enabled, created_by, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      `INSERT INTO runtime_pools (id, name, kind, placements, image, lifecycle, idle_minutes, slots_per_runtime, max_runtimes, min_runtimes, delivery, env, resources, cache, ttl_sec, enabled, created_by, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       [
         id,
         data.name,
@@ -331,6 +373,9 @@ export async function createRuntimePool(
         data.delivery,
         JSON.stringify(data.env),
         data.resources ? JSON.stringify(data.resources) : null,
+        data.cache
+          ? JSON.stringify({ ...data.cache, key: poolCacheKey(id, crypto.randomUUID()) })
+          : null,
         data.ttlSec,
         Number(data.enabled),
         actor,
@@ -364,8 +409,19 @@ export async function updateRuntimePool(
     await validatePool(data);
     if (await tx.get("SELECT id FROM runtime_pools WHERE name = ? AND id != ?", [data.name, id]))
       throw new RuntimePoolError("Pool name already exists", 409);
+    const cache = data.cache
+      ? {
+          ...data.cache,
+          key:
+            existing.cache && data.placements[0] === existing.placements[0]
+              ? (await getPoolCache(id))!.key
+              : poolCacheKey(id, crypto.randomUUID()),
+        }
+      : null;
+    if (existing.cache && (!data.cache || data.placements[0] !== existing.placements[0]))
+      await queuePoolCacheRelease(existing, actor);
     await tx.run(
-      `UPDATE runtime_pools SET name = ?, kind = ?, placements = ?, image = ?, resolved_image = ?, lifecycle = ?, idle_minutes = ?, slots_per_runtime = ?, max_runtimes = ?, min_runtimes = ?, delivery = ?, env = ?, resources = ?, ttl_sec = ?, enabled = ?, updated_by = COALESCE(?, updated_by), updated_at = ? WHERE id = ?`,
+      `UPDATE runtime_pools SET name = ?, kind = ?, placements = ?, image = ?, resolved_image = ?, lifecycle = ?, idle_minutes = ?, slots_per_runtime = ?, max_runtimes = ?, min_runtimes = ?, delivery = ?, env = ?, resources = ?, cache = ?, ttl_sec = ?, enabled = ?, updated_by = COALESCE(?, updated_by), updated_at = ? WHERE id = ?`,
       [
         data.name,
         data.kind,
@@ -384,6 +440,7 @@ export async function updateRuntimePool(
         data.delivery,
         JSON.stringify(data.env),
         data.resources ? JSON.stringify(data.resources) : null,
+        cache ? JSON.stringify(cache) : null,
         data.ttlSec,
         Number(data.enabled),
         actor,
@@ -400,7 +457,8 @@ export async function deleteRuntimePool(
   actor: string | null = null,
 ): Promise<void> {
   await getDbClient().transaction(async (tx) => {
-    if (!(await getRuntimePool(id))) throw new RuntimePoolError("Pool not found", 404);
+    const pool = await getRuntimePool(id);
+    if (!pool) throw new RuntimePoolError("Pool not found", 404);
     if (
       await tx.get(
         "SELECT id FROM runtime_instances WHERE pool_id = ? AND status != 'offline' LIMIT 1",
@@ -419,6 +477,7 @@ export async function deleteRuntimePool(
         [new Date().toISOString(), actor, id],
       );
     }
+    await queuePoolCacheRelease(pool, actor);
     await tx.run("DELETE FROM runtime_pools WHERE id = ?", [id]);
   });
 }

package/src/be/runtime-pools/remote-controller.ts

@@ -53,6 +53,7 @@ export class RemoteController implements ControllerCommands {
   readonly renew?: ControllerCommands["renew"];
   readonly pause?: ControllerCommands["pause"];
   readonly resume?: ControllerCommands["resume"];
+  readonly releaseCache?: ControllerCommands["releaseCache"];
 
   constructor(
     readonly id: string,
@@ -61,6 +62,10 @@ export class RemoteController implements ControllerCommands {
     readonly workerApiUrl?: string,
     private readonly timeouts: { commandTimeoutMs?: number; provisionTimeoutMs?: number } = {},
   ) {
+    if (traits.persistentCache)
+      this.releaseCache = async (cache) => {
+        await this.call({ op: "release-cache", payload: cache });
+      };
     if (traits.maxTtlSec)
       this.renew = async (handle, ttlSec) =>
         z

package/src/be/runtime-pools/supervisor.ts

@@ -29,6 +29,7 @@ import {
   runtimeStaleThresholdMinutes,
 } from "../multi-runtime";
 import { mintRuntimeToken } from "../users";
+import { getPoolCache, reconcileCacheReleases } from "./cache-cleanup";
 import {
   clearRuntimeOwnerIdCache,
   ensureRuntimeOwnerId,
@@ -245,6 +246,13 @@ export async function provisionRuntime(
     if (pool.lifecycle === "per-task") env.SWARM_EXIT_AFTER_TASKS = "1";
     if (templateId) env.TEMPLATE_ID = templateId;
     const ttlSec = pool.kind === "e2b" ? poolLeaseTtl(pool, controller) : undefined;
+    const currentPoolCache = await getPoolCache(pool.id);
+    const currentCache =
+      currentPoolCache?.controllerId === pool.placements[0] ? currentPoolCache : null;
+    const cache =
+      currentCache && controller.traits.persistentCache
+        ? { key: currentCache.key, mountPath: currentCache.mountPath }
+        : undefined;
     signal?.throwIfAborted();
     const handle = await controller.provision({
       runtimeId,
@@ -255,6 +263,7 @@ export async function provisionRuntime(
       ...(ttlSec === undefined ? {} : { ttlSec }),
       env,
       resources: pool.resources ?? undefined,
+      ...(cache ? { cache } : {}),
       labels: {
         [RUNTIME_LABELS.runtimeId]: runtimeId,
         [RUNTIME_LABELS.poolId]: pool.id,
@@ -267,6 +276,25 @@ export async function provisionRuntime(
       await markRuntimeTerminated(runtimeId, "Supervisor stopped during provision");
       return false;
     }
+    const cacheWarning = currentCache
+      ? !controller.traits.persistentCache
+        ? "Controller does not support persistent caches; runtime started without cache"
+        : handle.meta?.cacheWarning
+      : undefined;
+    // A cache is optional: surface degradation without counting a provision failure.
+    const cacheErrorPrefix = "Persistent cache: ";
+    const cacheError = cacheWarning ? cacheErrorPrefix + scrubSecrets(cacheWarning) : null;
+    await getDbClient().run(
+      `UPDATE runtime_pools SET last_error = ?, last_error_at = ?
+       WHERE id = ? AND (last_error IS NULL OR substr(last_error, 1, ?) = ?)`,
+      [
+        cacheError,
+        cacheError ? new Date().toISOString() : null,
+        pool.id,
+        cacheErrorPrefix.length,
+        cacheErrorPrefix,
+      ],
+    );
     await getDbClient().run(
       "UPDATE runtime_instances SET external_handle = ?, lease_expires_at = ?, created_at = COALESCE(?, created_at), updated_at = ? WHERE id = ?",
       [
@@ -886,11 +914,14 @@ export async function reconcileOnce(
   opts: { now?: Date; getControllerCommands?: typeof getControllerCommands } = {},
 ): Promise<void> {
   if (!isRuntimePoolsEnabled() || !isMultiRuntimeEnabled() || inFlight) return;
-  tickAbort = new AbortController();
+  const tick = new AbortController();
+  tickAbort = tick;
   inFlight = reconcilePools(
     opts.now ?? new Date(),
     opts.getControllerCommands ?? getControllerCommands,
-    tickAbort.signal,
+    tick.signal,
+  ).then(() =>
+    reconcileCacheReleases(opts.getControllerCommands ?? getControllerCommands, tick.signal),
   );
   try {
     await inFlight;

package/src/be/scripts/typecheck.ts

@@ -65,6 +65,8 @@ export type AgentTaskStepConfig = {
   parentTaskId?: string;
   requestedByUserId?: string;
   outputSchema?: Record<string, unknown>;
+  /** Dispatch even when the follow-up re-delegation guard blocks this step (a run launched from a follow-up task in a Slack thread, checked at its first new step). */
+  allowRedelegation?: boolean;
   /** Wait for the dispatched task to reach a terminal status before resolving. Default: true. */
   waitForCompletion?: boolean;
   /** Max ms to wait for a terminal status before throwing. Default: 2h. Only used when waitForCompletion is true. */
@@ -497,6 +499,26 @@ export interface ScriptWorkflowSteps {
     label: string,
     config: { prompt: string; model?: string; schema?: Record<string, unknown> },
   ): Promise<unknown>;
+  /**
+   * Dispatch a task and journal its result under \`label\`.
+   *
+   * Resolves to the task's OUTPUT on success: the stored output string (JSON
+   * text under \`outputSchema\`, so parse it yourself), or \`null\` when the task
+   * stored none. It is never a task record: on a string, \`result.taskId\` and
+   * \`result.status\` are undefined; on \`null\`, reading them throws a TypeError.
+   * Other shapes:
+   * - \`failOnTaskFailure: false\` and the task ends failed/cancelled/superseded:
+   *   \`{ taskId, status: "failed", error }\`.
+   * - \`waitForCompletion: false\`: the first call still waits up to ~30s while
+   *   the server polls the task. Done in that window: the output, as on
+   *   success. Failed or cancelled in that window: throws, or
+   *   \`{ taskId, status: "failed", error }\` with \`failOnTaskFailure: false\`.
+   *   Still running: \`{ taskId, status }\`.
+   *
+   * To get a step's task id, query by its context key:
+   * \`SELECT id, status, output FROM agent_tasks WHERE contextKey = 'script-run:<runId>:<label>'
+   * AND taskType = 'script-run-step' ORDER BY createdAt DESC, rowid DESC LIMIT 1\`.
+   */
   agentTask(
     label: string,
     config: AgentTaskStepConfig,

package/src/be/seed-scripts/catalog/task-failure-audit.ts

@@ -30,7 +30,11 @@ const REASON_PATTERNS: any[] = [
   { key: "context-window", re: /context (window|limit|saturat)|peakcontext|compact/i },
   { key: "reboot-sweep", re: /reboot sweep/i },
   { key: "not-found", re: /not found|404|missing|no such/i },
-  { key: "auth/credentials", re: /unauthorized|401|403|credential|token|forbidden/i },
+  {
+    key: "output-limit",
+    re: /\bmax_(?:output_)?tokens\b|\b(?:maximum|max) output (?:length|tokens)\b|\boutput (?:token|length) limit\b/i,
+  },
+  { key: "auth/credentials", re: /unauthorized|401|403|credential|\btoken\b|forbidden/i },
   { key: "ci/checks-failed", re: /\bci\b|check.?s? fail|lint|tsc|test.?s? fail/i },
   { key: "network", re: /network|econn|fetch failed|socket|dns|502|503|504/i },
   { key: "cancelled", re: /cancel|aborted/i },