@nylorun/runtime 0.42.0-beta → 0.44.0-beta

package/dist/store/postgres/schema.d.ts

@@ -4484,7 +4484,8 @@ export declare const turns: import("drizzle-orm/pg-core").PgTableWithColumns<{
 }>;
 /**
  * Each session's queue (protocol 17, D76): messages that arrived while its turn ran or waited, in
- * order (`seq`), until a turn takes them. A row holds the command as sent, the turn input its
+ * order (`seq`), until a turn takes them: the next turn, or the running one before a step for a
+ * steered message (D85). A row holds the command as sent, the turn input its
  * message gives the engine (file parts pinned when it was queued), and who sent it.
  */
 export declare const sessionQueue: import("drizzle-orm/pg-core").PgTableWithColumns<{
@@ -4679,6 +4680,23 @@ export declare const sessionQueue: import("drizzle-orm/pg-core").PgTableWithColu
         }, {}, {
             pgColumnBuilderBrand: 'PgCustomColumnBuilderBrand';
         }>;
+        delivery: import("drizzle-orm/pg-core").PgColumn<{
+            name: "delivery";
+            tableName: "session_queue";
+            dataType: "string";
+            columnType: "PgText";
+            data: "queue" | "steer";
+            driverParam: string;
+            notNull: true;
+            hasDefault: true;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: ["queue", "steer"];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}, {}>;
     };
     dialect: 'pg';
 }>;

package/dist/store/postgres/schema.js

@@ -560,7 +560,8 @@ export const turns = nylorun.table("turns", {
 ]);
 /**
  * Each session's queue (protocol 17, D76): messages that arrived while its turn ran or waited, in
- * order (`seq`), until a turn takes them. A row holds the command as sent, the turn input its
+ * order (`seq`), until a turn takes them: the next turn, or the running one before a step for a
+ * steered message (D85). A row holds the command as sent, the turn input its
  * message gives the engine (file parts pinned when it was queued), and who sent it.
  */
 export const sessionQueue = nylorun.table("session_queue", {
@@ -580,7 +581,15 @@ export const sessionQueue = nylorun.table("session_queue", {
     /** What sent it when not a client (track T4): a trigger's run, or the session's own wake. */
     cause: jsonText(),
     createdAt: textC().notNull(),
-}, (t) => [uniqueIndex("session_queue_order").on(t.sessionId, t.seq)]);
+    /**
+     * `steer` (D85): the running turn may take it before a step (the `inputs` effect); `queue`:
+     * it waits for the turn to end. Either starts the next turn if the running one didn't take it.
+     */
+    delivery: text({ enum: ["queue", "steer"] }).notNull().default("queue"),
+}, (t) => [
+    uniqueIndex("session_queue_order").on(t.sessionId, t.seq),
+    check("session_queue_delivery_check", sql `${t.delivery} IN ('queue', 'steer')`),
+]);
 /**
  * The Tool Gate's keyed MCP calls (F4.1 G3): one row per call, written by the gates service
  * before it calls the server and given the answer after. A re-send whose row has an answer

package/dist/store/postgres/store.js

@@ -1270,6 +1270,17 @@ class PostgresTx {
         if (ids.length > 0)
             await this.db.delete(sessionQueue).where(inArray(sessionQueue.id, [...ids]));
     }
+    async queuedSessions(options) {
+        this.check();
+        const rows = await this.db
+            .selectDistinct({ id: sessionQueue.sessionId })
+            .from(sessionQueue)
+            .innerJoin(sessions, eq(sessions.id, sessionQueue.sessionId))
+            .where(and(inArray(sessions.status, [...options.statuses]), options.after === undefined ? undefined : gt(sessionQueue.sessionId, options.after), options.environmentId === undefined ? undefined : eq(sessions.environmentId, options.environmentId)))
+            .orderBy(sessionQueue.sessionId)
+            .limit(options.limit);
+        return rows.map((row) => row.id);
+    }
     // --- triggers (track T4) ----------------------------------------------------
     async agentTriggers(agentId) {
         this.check();

package/dist/store/types.d.ts

@@ -733,8 +733,18 @@ export interface Tx {
     /** A session's queued messages, first to go first. */
     queuedMessages(sessionId: string): Promise<QueuedMessageRow[]>;
     countQueuedMessages(sessionId: string): Promise<number>;
-    /** Removes queued messages a turn took. */
+    /** Removes queued messages: a turn took them, or they failed or were cancelled. */
     deleteQueuedMessages(ids: readonly string[]): Promise<void>;
+    /**
+     * The ids of sessions with queued messages whose status is one of `statuses` (no turn holds
+     * them), by id after `after`, optionally on one Environment: queues a refused start left waiting.
+     */
+    queuedSessions(options: {
+        statuses: readonly string[];
+        limit: number;
+        after?: string;
+        environmentId?: string;
+    }): Promise<string[]>;
     agentTriggers(agentId: string): Promise<AgentTriggerRow[]>;
     /** One trigger's state; with `lock`, its row is locked until the transaction ends. */
     agentTrigger(agentId: string, triggerId: string, options?: {

package/dist/tenant/commands.d.ts

@@ -37,7 +37,10 @@ export interface MessageCause {
     triggerId: string;
     runId: string;
 }
-/** How a message was taken: it started a turn, or waits in the session's queue. */
+/**
+ * How a message was taken: it started a turn, or waits in the session's queue, marked with the
+ * delivery that applied (D85): `steer` when the running turn may take it before a step.
+ */
 export type MessageAnswer = {
     status: "accepted";
     turnId: string;
@@ -47,29 +50,71 @@ export type MessageAnswer = {
     queuedMessageId: string;
     position: number;
     cursor: string;
+    delivery: "queue" | "steer";
 };
 /**
  * Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
  * session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
- * an idle session whose queue was left waiting (its Agent was unavailable when the last turn
- * ended) starts a turn with all of them, this one last. Past `sessionQueuedMessages` it is
- * `429 queue_full`. `waiting` is the queue's length, read under the session's lock.
+ * an idle session whose queue was left waiting (its Agent refused the start when the last turn
+ * ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
+ * `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
+ * `steer` (D85) marks a waiting message so the running turn may take it before a step; a
+ * workflow session can't, so it waits as `queue`.
  */
-export declare function acceptMessage(ctx: TenantContext, t: Tx, s: Session, id: string, message: TurnMessage, waiting?: number): Promise<MessageAnswer>;
+export declare function acceptMessage(ctx: TenantContext, t: Tx, s: Session, id: string, message: TurnMessage, waiting?: number, asked?: "queue" | "steer"): Promise<MessageAnswer>;
+/**
+ * Takes the messages steered into session `s`'s running turn (D85), before step `step`: the
+ * `inputs` effect, in the transaction that journals it (`harness-api/record.ts`). Only `steer`
+ * rows whose turn request (variant, `removeTools`) is the running turn's join it; the rest wait
+ * for the turn's end. Writes each one's `command.message` under the turn, then one `queue.taken`
+ * with the step, adds their file artifacts to the session's, and returns their inputs (each as its
+ * message gave it, which the engine reads as a user message) in order. A workflow session takes
+ * none.
+ */
+export declare function takeSteeredMessages(t: Tx, s: Session, id: string, step: number): Promise<unknown[]>;
 /** Who checks a queued message's turn start: core, since the sender's grants were checked when it was queued. */
 export declare const QUEUE_SCOPE: AuthScope;
 /**
  * Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
- * wait (protocol 17, D76): in the transaction that ended the last turn, or a cancel's. Our engine
- * takes every queued message that carries the same turn manifest as the first, a workflow one.
- * Writes `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that
- * cannot take a turn now, whose messages wait for the next message or turn end.
+ * wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
+ * that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
+ * every queued message that carries the same turn manifest as the first, a workflow one. Writes
+ * `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
+ * the start.
+ *
+ * A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
+ * `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
+ * end of a disk copy try again. A message that has waited longer than the Runtime limit
+ * `queuedStartMs` since the stall began (or since it was queued, when later) fails with
+ * `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
+ * at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
+ * feature `queue-recovery`).
  */
-export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string): Promise<{
+export declare function startQueuedTurn(ctx: TenantContext, t: Tx, s: Session, id: string, now?: Date): Promise<{
     turnId: string;
     taken: string[];
     cursor: string;
 } | undefined>;
+/**
+ * Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
+ * `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
+ * never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
+ * turn took it, or it failed or was cancelled already).
+ */
+export declare function cancelQueuedMessage(ctx: TenantContext, id: string, queuedMessageId: string, scope: AuthScope, reason?: string): Promise<{
+    cursor: string;
+} | undefined>;
+/**
+ * Tries again the turn start of every queue a refused start left waiting, one transaction per
+ * session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
+ * disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
+ * `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
+ */
+export declare function retryQueuedStarts(ctx: TenantContext, options?: {
+    environmentId?: string;
+    now?: Date;
+    batchSize?: number;
+}): Promise<string[]>;
 /**
  * Who sent a command (R4.9), from its credential and never its body: a client over the Runtime
  * API (an application, or a person through it or a token), or core itself.

package/dist/tenant/commands.js

@@ -19,7 +19,12 @@
  * (`session_queue`, `message.queued`) unless it asks `delivery: "reject"`, which is
  * `409 session_busy`. A turn that ends takes every queued message in the same transaction
  * (`startQueuedTurn`): client messages and queued ones start a turn through one door,
- * `beginMessageTurn`, which writes each one's `command.message`.
+ * `beginMessageTurn`, which writes each one's `command.message`. A start the Agent refuses leaves
+ * the queue waiting, tried again by the Tenant sweep until the Runtime limit `queuedStartMs`, then
+ * failed (`queue.failed`, Host feature `queue-recovery`). A message that asks
+ * `delivery: "steer"` (D85) waits marked `steer`, and the running turn takes it before a step
+ * (`takeSteeredMessages`, the `inputs` effect); one it doesn't take starts the next turn like
+ * any other.
  */
 import { randomUUID } from "node:crypto";
 import { createDurableCheckpoint, createFlowCheckpoint, resumeFlowCheckpoint, } from "@nylorun/harness/run";
@@ -157,7 +162,7 @@ options = {}) {
                 // Every turn start on a sandbox: it exists, the token's grants reach it, and no other
                 // session's turn holds it. A queued message is checked when it is queued, too.
                 await checkAgentTurn(t, s, scope, ctx);
-                answer = await acceptMessage(ctx, t, s, id, await turnMessage(t, s, id, command, scope, options.trace), waiting);
+                answer = await acceptMessage(ctx, t, s, id, await turnMessage(t, s, id, command, scope, options.trace), waiting, delivery === "steer" ? "steer" : "queue");
             }
             else if (command.type === "approve" &&
                 (held = await heldCall(t, s, command.interactionId)) !== undefined) {
@@ -297,11 +302,13 @@ export const takesMessages = (s) => TAKES_MESSAGES.includes(s.status);
 /**
  * Takes a resolved message on session `s` (locked) in transaction `t`: it starts a turn when the
  * session takes messages and none wait, else it waits in the queue behind them (protocol 17), and
- * an idle session whose queue was left waiting (its Agent was unavailable when the last turn
- * ended) starts a turn with all of them, this one last. Past `sessionQueuedMessages` it is
- * `429 queue_full`. `waiting` is the queue's length, read under the session's lock.
+ * an idle session whose queue was left waiting (its Agent refused the start when the last turn
+ * ended, `startQueuedTurn`) tries the start again with all of them, this one last. Past `sessionQueuedMessages` it is
+ * `429 queue_full`. `waiting` is the queue's length, read under the session's lock. `asked`
+ * `steer` (D85) marks a waiting message so the running turn may take it before a step; a
+ * workflow session can't, so it waits as `queue`.
  */
-export async function acceptMessage(ctx, t, s, id, message, waiting) {
+export async function acceptMessage(ctx, t, s, id, message, waiting, asked = "queue") {
     const queued = waiting ?? (await t.countQueuedMessages(id));
     const takes = takesMessages(s);
     if (takes && queued === 0) {
@@ -313,6 +320,7 @@ export async function acceptMessage(ctx, t, s, id, message, waiting) {
     if (queued >= sessionQueuedMessages)
         fail(429, `Session ${id} already holds ${queued} queued messages, the Runtime limit sessionQueuedMessages (PUT /v1/tenant/limits); they start a turn when the running one ends.`, { code: "queue_full" });
     const queuedMessageId = randomUUID();
+    const delivery = asked === "steer" && !isWorkflowManifest(s.manifest) ? "steer" : "queue";
     await t.enqueueMessage({
         id: queuedMessageId,
         sessionId: id,
@@ -323,13 +331,56 @@ export async function acceptMessage(ctx, t, s, id, message, waiting) {
         trace: message.trace ?? null,
         cause: message.cause ?? null,
         createdAt: new Date().toISOString(),
+        delivery,
     });
     const position = queued + 1;
-    const event = await t.event(id, s.activeTurnId, "message.queued", { queuedMessageId, position, requestId: message.command.requestId }, message.writer);
+    const event = await t.event(id, s.activeTurnId, "message.queued", { queuedMessageId, position, requestId: message.command.requestId, delivery }, message.writer);
     const started = takes ? await startQueuedTurn(ctx, t, s, id) : undefined;
     if (started?.taken.includes(queuedMessageId))
         return { status: "accepted", turnId: started.turnId, cursor: started.cursor };
-    return { status: "queued", queuedMessageId, position, cursor: event.cursor };
+    return { status: "queued", queuedMessageId, position, cursor: event.cursor, delivery };
+}
+/**
+ * Takes the messages steered into session `s`'s running turn (D85), before step `step`: the
+ * `inputs` effect, in the transaction that journals it (`harness-api/record.ts`). Only `steer`
+ * rows whose turn request (variant, `removeTools`) is the running turn's join it; the rest wait
+ * for the turn's end. Writes each one's `command.message` under the turn, then one `queue.taken`
+ * with the step, adds their file artifacts to the session's, and returns their inputs (each as its
+ * message gave it, which the engine reads as a user message) in order. A workflow session takes
+ * none.
+ */
+export async function takeSteeredMessages(t, s, id, step) {
+    const turnId = s.activeTurnId;
+    if (turnId === null || isWorkflowManifest(s.manifest))
+        return [];
+    const running = s.turnRequest?.turnId === turnId ? s.turnRequest : undefined;
+    if (running?.manifest !== undefined)
+        return [];
+    const runningTools = canonical([...new Set(running?.removeTools ?? [])].sort());
+    const rows = (await t.queuedMessages(id)).filter((row) => {
+        if (row.delivery !== "steer")
+            return false;
+        const request = turnManifestRequestOf(row.command);
+        if (request?.manifest !== undefined)
+            return false;
+        return canonical([...(request?.removeTools ?? [])].sort()) === runningTools;
+    });
+    if (rows.length === 0)
+        return [];
+    const taken = rows.map((row) => row.id);
+    await t.deleteQueuedMessages(taken);
+    const named = rows.flatMap((row) => row.fileArtifacts);
+    if (named.length > 0)
+        s.fileArtifacts = [...new Set([...(s.fileArtifacts ?? []), ...named])];
+    for (const row of rows)
+        await t.event(id, turnId, "command.message", {
+            ...row.command,
+            queuedMessageId: row.id,
+            ...(row.trace ? { trace: row.trace } : {}),
+            ...(row.cause ? { cause: row.cause } : {}),
+        }, row.writer);
+    await t.event(id, turnId, "queue.taken", { turnId, queuedMessageIds: taken, step });
+    return rows.map((row) => row.input);
 }
 /**
  * Resolves a message for a turn, now or when it leaves the queue: file parts name artifacts the
@@ -415,6 +466,8 @@ async function beginMessageTurn(t, s, id, messages) {
         });
     }
     s.turnStartState = s.state;
+    // A turn holds the session now: its queue, if any, waits for the turn, not for the Agent.
+    delete s.queueStalled;
     // A new turn folds from the latest snapshot: a cancel or failure only reverts this turn.
     if (s.history?.snapshot !== undefined)
         s.history = { ...s.history, from: s.history.snapshot };
@@ -450,52 +503,166 @@ async function beginMessageTurn(t, s, id, messages) {
 }
 /** Who checks a queued message's turn start: core, since the sender's grants were checked when it was queued. */
 export const QUEUE_SCOPE = { kind: "application", principalId: "queue" };
+/**
+ * The refusals of a queued turn's start that a later try may get past (Host feature
+ * `queue-recovery`): the Agent's disk is being copied or it is being deleted (`agent_busy`), it
+ * can't run now or was deleted and may be made again (`agent_unavailable`), its pod was lost and
+ * may be reset (`agent_lost`). Any other refusal (a turn request the version no longer resolves,
+ * another harness, another owner's Agent) fails the messages at once.
+ */
+const RETRIED_REFUSALS = new Set(["agent_busy", "agent_unavailable", "agent_lost"]);
+const retried = (error) => error.status === 429 ||
+    error.status === 503 ||
+    (error.status === 409 && RETRIED_REFUSALS.has(error.rejection.code ?? ""));
 /**
  * Starts the next turn of session `s` (locked) from its queue, when it takes messages and messages
- * wait (protocol 17, D76): in the transaction that ended the last turn, or a cancel's. Our engine
- * takes every queued message that carries the same turn manifest as the first, a workflow one.
- * Writes `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that
- * cannot take a turn now, whose messages wait for the next message or turn end.
+ * wait (protocol 17, D76): in the transaction that ended the last turn, a cancel's, a message's
+ * that joined a queue left waiting, or the Tenant sweep's (`retryQueuedStarts`). Our engine takes
+ * every queued message that carries the same turn manifest as the first, a workflow one. Writes
+ * `queue.taken`. Returns nothing when nothing started: an empty queue, or an Agent that refused
+ * the start.
+ *
+ * A refusal a later try may get past (`RETRIED_REFUSALS`) leaves the messages waiting, with
+ * `s.queueStalled` saying since when and why (written here); the next message, the sweep and the
+ * end of a disk copy try again. A message that has waited longer than the Runtime limit
+ * `queuedStartMs` since the stall began (or since it was queued, when later) fails with
+ * `queue.failed` (`queue.start_expired`); a refusal no retry changes fails the messages it refused
+ * at once (`queue.start_refused`). Either way the messages behind them are tried next (Host
+ * feature `queue-recovery`).
  */
-export async function startQueuedTurn(ctx, t, s, id) {
+export async function startQueuedTurn(ctx, t, s, id, now = new Date()) {
     if (s.closedReason !== undefined || !takesMessages(s) || s.activeTurnId !== null)
         return undefined;
-    const queued = await t.queuedMessages(id);
-    if (queued.length === 0)
-        return undefined;
-    // Messages that ask the same of their turn's manifest (a variant, `removeTools`) share a turn.
-    const manifestOf = (row) => canonical(turnManifestRequestOf(row.command) ?? null);
-    const batch = isWorkflowManifest(s.manifest) ? [queued[0]] : [];
-    if (batch.length === 0)
-        for (const row of queued) {
-            if (manifestOf(row) !== manifestOf(queued[0]))
-                break;
-            batch.push(row);
+    for (;;) {
+        const queued = await t.queuedMessages(id);
+        if (queued.length === 0) {
+            if (s.queueStalled !== undefined) {
+                delete s.queueStalled;
+                await t.put("sessions", id, s);
+            }
+            return undefined;
         }
-    try {
-        await checkAgentTurn(t, s, QUEUE_SCOPE, ctx);
-        await beginMessageTurn(t, s, id, batch.map((row) => ({
-            command: row.command,
-            input: row.input,
-            fileArtifacts: row.fileArtifacts,
-            writer: row.writer,
-            queuedMessageId: row.id,
-            ...(row.trace ? { trace: row.trace } : {}),
-            ...(row.cause ? { cause: row.cause } : {}),
-        })));
-    }
-    catch (error) {
-        if (error instanceof HttpError)
+        // Messages that ask the same of their turn's manifest (a variant, `removeTools`) share a turn.
+        const manifestOf = (row) => canonical(turnManifestRequestOf(row.command) ?? null);
+        const batch = isWorkflowManifest(s.manifest) ? [queued[0]] : [];
+        if (batch.length === 0)
+            for (const row of queued) {
+                if (manifestOf(row) !== manifestOf(queued[0]))
+                    break;
+                batch.push(row);
+            }
+        try {
+            await checkAgentTurn(t, s, QUEUE_SCOPE, ctx);
+            await beginMessageTurn(t, s, id, batch.map((row) => ({
+                command: row.command,
+                input: row.input,
+                fileArtifacts: row.fileArtifacts,
+                writer: row.writer,
+                queuedMessageId: row.id,
+                ...(row.trace ? { trace: row.trace } : {}),
+                ...(row.cause ? { cause: row.cause } : {}),
+            })));
+        }
+        catch (error) {
+            if (!(error instanceof HttpError))
+                throw error;
+            const reason = error.rejection.code ?? "request_rejected";
+            if (!retried(error)) {
+                await failQueued(t, id, batch, {
+                    code: "queue.start_refused",
+                    message: `The Agent refused these messages' turn: ${error.message}`,
+                    reason,
+                });
+                continue;
+            }
+            const since = s.queueStalled?.since ?? now.toISOString();
+            const { queuedStartMs } = await readRuntimeLimits(t);
+            const waited = (row) => now.getTime() - Math.max(Date.parse(since), Date.parse(row.createdAt));
+            const expired = batch.filter((row) => waited(row) >= queuedStartMs);
+            if (expired[0]?.id === batch[0].id) {
+                await failQueued(t, id, expired, {
+                    code: "queue.start_expired",
+                    message: `The Agent refused these messages' turn for longer than the Runtime limit queuedStartMs (${queuedStartMs} ms): ${error.message}`,
+                    reason,
+                });
+                continue;
+            }
+            s.queueStalled = { since, code: reason, message: error.message };
+            await t.put("sessions", id, s);
             return undefined;
-        throw error;
+        }
+        const taken = batch.map((row) => row.id);
+        await t.deleteQueuedMessages(taken);
+        const event = await t.event(id, s.activeTurnId, "queue.taken", {
+            turnId: s.activeTurnId,
+            queuedMessageIds: taken,
+        });
+        return { turnId: s.activeTurnId, taken, cursor: event.cursor };
     }
-    const taken = batch.map((row) => row.id);
-    await t.deleteQueuedMessages(taken);
-    const event = await t.event(id, s.activeTurnId, "queue.taken", {
-        turnId: s.activeTurnId,
-        queuedMessageIds: taken,
+}
+/** Removes queued messages no turn will take, with `queue.failed` (Host feature `queue-recovery`). */
+async function failQueued(t, id, rows, error) {
+    const ids = rows.map((row) => row.id);
+    await t.deleteQueuedMessages(ids);
+    await t.event(id, null, "queue.failed", { queuedMessageIds: ids, error });
+}
+/**
+ * Removes queued message `queuedMessageId` from session `id`'s queue, as `scope` (Host feature
+ * `queue-recovery`): `queue.cancelled`, written by the caller. A turn running or not, the message
+ * never starts one. Returns the event's cursor, or undefined when the message no longer waits (a
+ * turn took it, or it failed or was cancelled already).
+ */
+export async function cancelQueuedMessage(ctx, id, queuedMessageId, scope, reason) {
+    return ctx.store.tx(async (t) => {
+        const s = await lockedSession(t, id, accessOf(scope));
+        const queued = await t.queuedMessages(id);
+        if (!queued.some((row) => row.id === queuedMessageId))
+            return undefined;
+        await t.deleteQueuedMessages([queuedMessageId]);
+        const event = await t.event(id, null, "queue.cancelled", { queuedMessageIds: [queuedMessageId], ...(reason === undefined ? {} : { reason }) }, commandWriter(scope));
+        if (queued.length === 1 && s.queueStalled !== undefined) {
+            delete s.queueStalled;
+            await t.put("sessions", id, s);
+        }
+        return { cursor: event.cursor };
     });
-    return { turnId: s.activeTurnId, taken, cursor: event.cursor };
+}
+/**
+ * Tries again the turn start of every queue a refused start left waiting, one transaction per
+ * session (Host feature `queue-recovery`): the Tenant sweep's step, and what the end of an Agent's
+ * disk copy runs for that Agent's sessions (`environmentId`). Each start fails its messages past
+ * `queuedStartMs` (`startQueuedTurn`). Returns the sessions whose queue started a turn.
+ */
+export async function retryQueuedStarts(ctx, options = {}) {
+    const started = [];
+    const limit = options.batchSize ?? 100;
+    let after;
+    for (;;) {
+        const ids = await ctx.store.tx((t) => t.queuedSessions({
+            statuses: TAKES_MESSAGES,
+            limit,
+            ...(after === undefined ? {} : { after }),
+            ...(options.environmentId === undefined ? {} : { environmentId: options.environmentId }),
+        }));
+        for (const id of ids) {
+            if (ctx.closing || ctx.closed)
+                return started;
+            const turn = await ctx.store.tx(async (t) => {
+                const s = await t.lockSession(id);
+                if (!s)
+                    return undefined;
+                const begun = await startQueuedTurn(ctx, t, s, id, options.now ?? new Date());
+                if (begun)
+                    await t.put("sessions", id, s);
+                return begun;
+            });
+            if (turn)
+                started.push(id);
+        }
+        if (ids.length < limit)
+            return started;
+        after = ids.at(-1);
+    }
 }
 /** Core's own commands (R4.9): the flow host's, and a drain's cancels. */
 const CORE_PRINCIPALS = { "flow-host": "flow-host", drain: "core" };

package/dist/tenant/context.d.ts

@@ -113,6 +113,17 @@ export interface Session {
         triggerId: string;
         runId: string;
     };
+    /**
+     * Set while the session's queued messages wait for an Agent that refused their turn's start
+     * (Host feature `queue-recovery`, `startQueuedTurn`): since when, and the last refusal. The
+     * Tenant sweep tries the start again, and fails them once `queuedStartMs` has passed since.
+     * Cleared when a turn starts, or when the queue empties.
+     */
+    queueStalled?: {
+        since: string;
+        code: string;
+        message: string;
+    };
     creation: unknown;
     mcpSnapshot?: McpSnapshot;
     mcpDiagnostics?: readonly McpDiagnostic[];