@agent-native/core 0.220.0-nightly-20261024033422 → 0.220.0

package/dist/agent/run-store.js

@@ -1,6 +1,6 @@
 import { MAX_BACKGROUND_RUN_CONTINUATIONS, TURN_RUN_LEDGER_SLACK, } from "../app-config/run-lifecycle-invariants.js";
 import { isArtifactReceipt, } from "../artifacts/detect.js";
-import { getDbExec } from "../db/client.js";
+import { getDbExec, getScopedTransactionDbExec } from "../db/client.js";
 import { ensureColumnExists, ensureTableExists } from "../db/ddl-guard.js";
 import { defineStore } from "../db/store-registry.js";
 import { fromMigrationEntries } from "../db/versioned-store-migrations.js";
@@ -192,7 +192,9 @@ export const agentRunsStore = defineStore({
           dispatch_payload TEXT,
           peak_rss_mb BIGINT,
           continuation_order BIGINT,
-          thread_projection_pending BOOLEAN NOT NULL DEFAULT FALSE
+          thread_projection_pending BOOLEAN NOT NULL DEFAULT FALSE,
+          thread_projection_retry_order BIGINT,
+          thread_projection_error_code TEXT
         )
       `;
                 const agentRunEventsCreateSql = `
@@ -257,6 +259,8 @@ export const agentRunsStore = defineStore({
                     ["continuation_order", "BIGINT"],
                     ["auto_continue_of", "TEXT"],
                     ["thread_projection_pending", "BOOLEAN NOT NULL DEFAULT FALSE"],
+                    ["thread_projection_retry_order", "BIGINT"],
+                    ["thread_projection_error_code", "TEXT"],
                 ]) {
                     await ensureColumnExists("agent_runs", col, `ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS ${col} ${colType}`);
                 }
@@ -286,6 +290,22 @@ export const agentRunsStore = defineStore({
             },
         },
         ...fromMigrationEntries(AGENT_RUN_MIGRATIONS_TABLE, AGENT_RUN_MIGRATIONS),
+        {
+            name: "thread-projection-retry-quarantine-runtime-schema",
+            run: async (exec) => {
+                await exec.execute({
+                    sql: "ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS thread_projection_error_code TEXT",
+                });
+                await exec.execute({
+                    sql: "CREATE SEQUENCE IF NOT EXISTS agent_run_projection_retry_order_seq",
+                });
+                await exec.execute({
+                    sql: `CREATE INDEX IF NOT EXISTS agent_runs_thread_projection_retry_order_idx
+                ON agent_runs (thread_projection_retry_order ASC NULLS FIRST, completed_at ASC, id ASC)
+                WHERE thread_projection_pending IS TRUE`,
+                });
+            },
+        },
     ],
 });
 export function ensureRunTables() {
@@ -613,6 +633,101 @@ export async function markAgentExecutionStepDispatched(intent) {
         throw new Error(`Execution step ${intent.stepId} was not prepared`);
     }
 }
+export async function resetAgentExecutionStepBeforeInvocation(intent) {
+    await ensureRunTables();
+    const expectedFence = {
+        threadId: intent.threadId,
+        turnId: intent.turnId,
+        runId: intent.runId,
+        generation: intent.generation,
+    };
+    try {
+        const result = await withAgentExecutionStepIntentFence(intent, (tx) => tx.execute({
+            sql: `UPDATE agent_execution_steps
+              SET status = 'prepared', dispatched_at = NULL,
+                  updated_at = clock_timestamp()
+              WHERE thread_id = ? AND turn_id = ? AND step_id = ?
+                AND run_id = ? AND generation = CAST(? AS BIGINT)
+                AND status IN ('prepared', 'dispatched')`,
+            args: [
+                intent.threadId,
+                intent.turnId,
+                intent.stepId,
+                intent.runId,
+                intent.generation,
+            ],
+        }));
+        if (result.rowsAffected !== 1) {
+            throw new Error(`Execution step ${intent.stepId} was not reset before invocation`);
+        }
+        return;
+    }
+    catch (error) {
+        if (!(error instanceof StaleExecutionFenceError))
+            throw error;
+        const activeFence = currentExecutionFence();
+        if (!activeFence ||
+            activeFence.threadId !== expectedFence.threadId ||
+            activeFence.turnId !== expectedFence.turnId ||
+            activeFence.runId !== expectedFence.runId ||
+            activeFence.generation !== expectedFence.generation) {
+            throw error;
+        }
+    }
+    const reconcileWithCurrentLease = async (tx) => {
+        await lockAndAssertCurrentExecutionFences(tx, expectedFence);
+        const currentLease = await lockTurnExecutionLease(tx, intent.threadId, intent.turnId);
+        if (!currentLease?.isLive) {
+            throw new StaleExecutionFenceError(expectedFence);
+        }
+        const currentOwner = await tx.execute({
+            sql: `SELECT status, worker_generation FROM agent_runs
+            WHERE id = ? FOR UPDATE`,
+            args: [currentLease.ownerRunId],
+        });
+        const owner = currentOwner.rows[0];
+        const currentFence = {
+            threadId: intent.threadId,
+            turnId: intent.turnId,
+            runId: currentLease.ownerRunId,
+            generation: currentLease.generation,
+        };
+        if (owner?.status !== "running" ||
+            String(owner.worker_generation) !== currentLease.generation) {
+            throw new StaleExecutionFenceError(currentFence);
+        }
+        const reconciled = await tx.execute({
+            sql: `UPDATE agent_execution_steps
+            SET status = 'prepared', run_id = ?,
+                generation = CAST(? AS BIGINT), dispatched_at = NULL,
+                updated_at = clock_timestamp()
+            WHERE thread_id = ? AND turn_id = ? AND step_id = ?
+              AND run_id = ? AND generation = CAST(? AS BIGINT)
+              AND status IN ('prepared', 'dispatched', 'unknown')`,
+            args: [
+                currentLease.ownerRunId,
+                currentLease.generation,
+                intent.threadId,
+                intent.turnId,
+                intent.stepId,
+                intent.runId,
+                intent.generation,
+            ],
+        });
+        if (reconciled.rowsAffected !== 1) {
+            throw new Error(`Execution step ${intent.stepId} was not reconciled under the current lease`);
+        }
+    };
+    const client = getDbExec();
+    if (getScopedTransactionDbExec() === client) {
+        await reconcileWithCurrentLease(client);
+        return;
+    }
+    if (!client.transaction) {
+        throw new Error("Execution step reconciliation requires transaction support");
+    }
+    await client.transaction(reconcileWithCurrentLease);
+}
 export async function completeAgentExecutionStep(intent, resultJson, errorJson) {
     await ensureRunTables();
     const result = await withFencedExecutionWrite((tx) => tx.execute({
@@ -714,16 +829,28 @@ export async function getCompletedAgentExecutionSteps(threadId, turnId, stepIds)
               AND step_id IN (${batch.map(() => "?").join(", ")})`,
             args: [threadId, turnId, ...batch],
         });
-        for (const raw of rows.rows) {
+        for (const [batchRowIndex, raw] of rows.rows.entries()) {
+            const rowIndex = offset + batchRowIndex;
+            if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+                throw new AgentRunJournalUnreadableError(threadId, turnId, rowIndex, "invalid_execution_step", {
+                    cause: new Error("Completed execution-step lookup returned an invalid row"),
+                });
+            }
             const row = raw;
             if (typeof row.step_id !== "string" || !batch.includes(row.step_id)) {
-                throw new Error("Completed execution-step lookup returned an invalid step id");
+                throw new AgentRunJournalUnreadableError(threadId, turnId, rowIndex, "invalid_execution_step", {
+                    cause: new Error("Completed execution-step lookup returned an invalid step id"),
+                });
             }
             if (typeof row.operation_id !== "string" ||
+                row.operation_id.trim().length === 0 ||
                 typeof row.tool_name !== "string" ||
+                row.tool_name.trim().length === 0 ||
                 typeof row.input_json !== "string" ||
                 typeof row.result_json !== "string") {
-                throw new Error(`Completed execution step ${row.step_id} has an incomplete stored intent or result`);
+                throw new AgentRunJournalUnreadableError(threadId, turnId, rowIndex, "invalid_execution_step", {
+                    cause: new Error(`Completed execution step ${row.step_id} has an incomplete stored intent or result`),
+                });
             }
             completed.push({
                 stepId: row.step_id,
@@ -986,7 +1113,7 @@ async function lockContinuationOrder(db, threadId, turnId) {
 async function nextContinuationOrder(db, threadId, turnId) {
     await lockContinuationOrder(db, threadId, turnId);
     const { rows } = await db.execute({
-        sql: `SELECT MAX(continuation_order) AS max_order
+        sql: `SELECT MAX(COALESCE(continuation_order, 0)) AS max_order
           FROM agent_runs
           WHERE thread_id = ? AND COALESCE(turn_id, id) = ?`,
         args: [threadId, turnId],
@@ -1056,6 +1183,31 @@ function terminalRunEventExclusionSql(runIdColumn = "id") {
 function livenessBasisSql() {
     return `(CASE WHEN COALESCE(last_progress_at, started_at) > COALESCE(heartbeat_at, started_at) THEN COALESCE(last_progress_at, started_at) ELSE COALESCE(heartbeat_at, started_at) END)`;
 }
+function earlierTurnProjectionPendingSql() {
+    return `NOT EXISTS (
+    SELECT 1 FROM agent_runs pending_projection
+    WHERE pending_projection.thread_id = agent_runs.thread_id
+      AND COALESCE(pending_projection.turn_id, pending_projection.id) =
+          COALESCE(agent_runs.turn_id, agent_runs.id)
+      AND pending_projection.id <> agent_runs.id
+      AND pending_projection.thread_projection_pending IS TRUE
+      AND (
+        COALESCE(pending_projection.continuation_order, 0) <
+            COALESCE(agent_runs.continuation_order, 0)
+        OR (
+          COALESCE(pending_projection.continuation_order, 0) =
+              COALESCE(agent_runs.continuation_order, 0)
+          AND (
+            pending_projection.started_at < agent_runs.started_at
+            OR (
+              pending_projection.started_at = agent_runs.started_at
+              AND pending_projection.id < agent_runs.id
+            )
+          )
+        )
+      )
+  )`;
+}
 export async function claimBackgroundRun(runId) {
     await ensureRunTables();
     const client = getDbExec();
@@ -1064,7 +1216,8 @@ export async function claimBackgroundRun(runId) {
           SET dispatch_mode = 'background-processing'
           WHERE id = ?
             AND status = 'running'
-            AND dispatch_mode = 'background'`,
+            AND dispatch_mode = 'background'
+            AND ${earlierTurnProjectionPendingSql()}`,
         args: [runId],
     });
     return (rowsAffected ?? 0) > 0;
@@ -1115,6 +1268,7 @@ export async function listUnclaimedBackgroundRunIds() {
         sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
             AND dispatch_mode = 'background'
+            AND ${earlierTurnProjectionPendingSql()}
             AND COALESCE(heartbeat_at, started_at) < (CAST(? AS BIGINT) - ${UNCLAIMED_BACKGROUND_RUN_GRACE_MS})`,
         args: [Date.now()],
     });
@@ -1140,6 +1294,7 @@ export async function listUnclaimedBackgroundRunRows(options) {
         sql: `SELECT id, started_at, (dispatch_payload IS NOT NULL) AS has_dispatch_payload FROM agent_runs
           WHERE status = 'running'
             AND dispatch_mode = 'background'
+            AND ${earlierTurnProjectionPendingSql()}
             AND COALESCE(heartbeat_at, started_at) < (CAST(? AS BIGINT) - ${UNCLAIMED_BACKGROUND_RUN_GRACE_MS})
           ORDER BY COALESCE(heartbeat_at, started_at) ASC, started_at ASC
           LIMIT ${limit}`,
@@ -1391,6 +1546,15 @@ export async function tryClaimRunSlot(threadId, runId, maxStaleMs, options) {
                 };
             }
         }
+        if (await options?.hasPendingApproval?.(tx)) {
+            return {
+                result: {
+                    claimed: false,
+                    activeRunId: null,
+                    approvalPending: true,
+                },
+            };
+        }
         // Admitted under the thread's slot lock, so two tabs continuing the same
         // stop start one run and the count of continuations stays exact.
         if (continueOf) {
@@ -1723,6 +1887,7 @@ export async function reconcileTerminalRunFromEvents(runId, options = {}) {
               error_code = ?,
               error_detail = ?,
               terminal_reason = ?,
+              thread_projection_retry_order = CASE WHEN ? THEN COALESCE(thread_projection_retry_order, nextval('agent_run_projection_retry_order_seq')) ELSE thread_projection_retry_order END,
               thread_projection_pending = CASE WHEN ? THEN TRUE ELSE thread_projection_pending END
           WHERE id = ?
             AND (
@@ -1737,6 +1902,7 @@ export async function reconcileTerminalRunFromEvents(runId, options = {}) {
             errorDetail,
             terminalReason,
             options.threadProjectionPending === true,
+            options.threadProjectionPending === true,
             runId,
             STALE_RUN_ERROR_EVENT.errorCode,
             ...(allowMatchingTerminalRepair ? [status, terminalReason] : []),
@@ -2043,6 +2209,7 @@ async function reconcileTerminalRunInRecoveryTransaction(tx, runId) {
               error_detail = ?,
               terminal_reason = ?
           WHERE id = ? AND status = 'running'
+            AND ${earlierTurnProjectionPendingSql()}
           ${ENDED_RUN_RETURNING}`,
         args: [
             status,
@@ -2075,6 +2242,7 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
               terminal_reason = ?
           WHERE id = ?
             AND status = 'running'
+            AND ${earlierTurnProjectionPendingSql()}
             AND ${terminalRunEventExclusionSql()}
             AND ${staleClause}
           ${ENDED_RUN_RETURNING}`;
@@ -2127,15 +2295,31 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
                     args: [runScope.thread_id, runScope.turn_id, runId, lease.generation],
                 });
             }
-            if (generationMatches || (!lease && run.worker_generation == null)) {
-                await tx.execute({
+            const executionLeaseChanged = !generationMatches && (lease !== null || run.worker_generation != null);
+            const shouldProjectRepairedRun = generationMatches || (!lease && run.worker_generation == null);
+            const outcome = isContinuationTerminalReason(repaired.ending.terminalReason) &&
+                !executionLeaseChanged
+                ? await attemptStaleRunRecovery(tx, runId)
+                : null;
+            if (shouldProjectRepairedRun) {
+                const projection = await tx.execute({
                     sql: `UPDATE agent_runs
-                SET thread_projection_pending = TRUE
-                WHERE id = ? AND status = 'errored'`,
-                    args: [runId],
+                SET thread_projection_pending = TRUE,
+                    thread_projection_retry_order = COALESCE(thread_projection_retry_order, nextval('agent_run_projection_retry_order_seq'))
+                WHERE id = ? AND status = ?
+                RETURNING id`,
+                    args: [runId, repaired.ending.status],
                 });
+                if ((projection.rowsAffected ?? projection.rows.length) !== 1) {
+                    throw new Error(`Repaired run ${runId} did not retain its thread projection retry marker`);
+                }
             }
-            return { reaped: false, outcome: null, repaired };
+            return {
+                reaped: false,
+                outcome,
+                repaired,
+                shouldProjectRepairedRun,
+            };
         }
         const { rows, rowsAffected } = await tx.execute({
             sql: updateSql,
@@ -2161,7 +2345,8 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
             if (outcome.outcome !== "execution_lease_changed") {
                 const projection = await tx.execute({
                     sql: `UPDATE agent_runs
-                SET thread_projection_pending = TRUE
+                SET thread_projection_pending = TRUE,
+                    thread_projection_retry_order = COALESCE(thread_projection_retry_order, nextval('agent_run_projection_retry_order_seq'))
                 WHERE id = ? AND status = 'errored'
                 RETURNING id`,
                     args: [runId],
@@ -2248,11 +2433,16 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
     // first would have this older run's steps land after its, and take its
     // error and status.
     let projectionFailed = false;
-    if (reaped) {
+    if ("repaired" in reapResult && reapResult.shouldProjectRepairedRun) {
+        projectionFailed = !(await finalizeStaleRun(runId, source));
+    }
+    else if (reaped) {
         // A newer generation owns the turn, so it must remain the sole thread projector.
         projectionFailed = !(await finalizeStaleRun(runId, source, outcome?.outcome !== "execution_lease_changed"));
     }
-    if (reaped && outcome && outcome.outcome !== "not_background") {
+    if ((reaped || "repaired" in reapResult) &&
+        outcome &&
+        outcome.outcome !== "not_background") {
         const outcomeDetail = outcome.outcome === "recovered"
             ? `recovered successorRunId=${outcome.successorRunId}`
             : `declined reason=${outcome.outcome}`;
@@ -2260,7 +2450,7 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
             ? `${outcomeDetail} ${priorStageInfo}`
             : outcomeDetail;
         await recordRunDiagnostic(runId, RUN_DIAG_STAGE.staleRunRecoveryAttempted, forensics ? `${detail} ${forensics}` : detail).catch(() => { });
-        if (outcome.outcome === "recovered") {
+        if (outcome.outcome === "recovered" && !projectionFailed) {
             attemptStaleRunRecoveryDispatch(outcome.successorRunId);
         }
     }
@@ -2298,6 +2488,7 @@ export async function reapUnclaimedBackgroundRun(runId) {
               AND status = 'running'
               AND dispatch_mode = 'background'
               AND COALESCE(heartbeat_at, started_at) < ?
+              AND ${earlierTurnProjectionPendingSql()}
             ${ENDED_RUN_RETURNING}`,
             args: [
                 completedAt,
@@ -2347,6 +2538,7 @@ export async function updateRunStatusIfRunning(runId, status, options = {}) {
     const terminalReason = options.terminalReason?.slice(0, 200);
     const { rowsAffected } = await withFencedExecutionWrite((tx) => tx.execute({
         sql: `UPDATE agent_runs SET status = ?, completed_at = ?, dispatch_payload = NULL,
+                     thread_projection_retry_order = CASE WHEN ? THEN COALESCE(thread_projection_retry_order, nextval('agent_run_projection_retry_order_seq')) ELSE thread_projection_retry_order END,
                      thread_projection_pending = CASE WHEN ? THEN TRUE ELSE thread_projection_pending END
                      ${terminalReason === undefined ? "" : ", terminal_reason = ?"}
               WHERE id = ? AND status = 'running'`,
@@ -2354,6 +2546,7 @@ export async function updateRunStatusIfRunning(runId, status, options = {}) {
             status,
             Date.now(),
             options.threadProjectionPending === true,
+            options.threadProjectionPending === true,
             ...(terminalReason === undefined ? [] : [terminalReason]),
             runId,
         ],
@@ -3279,7 +3472,7 @@ export async function getRunById(runId) {
     await ensureRunTables();
     const client = getDbExec();
     const { rows } = await client.execute({
-        sql: `SELECT id, thread_id, status, started_at, error_code, error_detail, terminal_reason FROM agent_runs WHERE id = ?`,
+        sql: `SELECT id, thread_id, status, started_at, error_code, error_detail, terminal_reason, thread_projection_error_code FROM agent_runs WHERE id = ?`,
         args: [runId],
     });
     if (rows.length === 0)
@@ -3293,6 +3486,7 @@ export async function getRunById(runId) {
         errorCode: r.error_code ?? null,
         errorDetail: r.error_detail ?? null,
         terminalReason: r.terminal_reason ?? null,
+        threadProjectionErrorCode: r.thread_projection_error_code ?? null,
     };
 }
 /**
@@ -3328,8 +3522,21 @@ export async function readStoppedRunForThreadFold(runId) {
     if (!row || row.status === "running")
         return null;
     const events = [];
-    for (const { seq, eventData } of await getRunEventsSince(runId, 0)) {
-        events.push({ seq, event: JSON.parse(eventData) });
+    for (const [rowIndex, { seq, eventData }] of (await getRunEventsSince(runId, 0)).entries()) {
+        if (!Number.isSafeInteger(seq) || seq < 0) {
+            throw new AgentRunJournalUnreadableError(row.thread_id, row.turn_id, rowIndex, "invalid_row");
+        }
+        let event;
+        try {
+            event = JSON.parse(eventData);
+        }
+        catch (cause) {
+            throw new AgentRunJournalUnreadableError(row.thread_id, row.turn_id, rowIndex, "invalid_event_json", { cause });
+        }
+        if (!isReadableJournalEvent(event)) {
+            throw new AgentRunJournalUnreadableError(row.thread_id, row.turn_id, rowIndex, "invalid_event");
+        }
+        events.push({ seq, event });
     }
     return {
         threadId: row.thread_id,
@@ -3481,7 +3688,7 @@ export async function listRunsForThread(threadId, options = {}) {
             ? ` AND id IN (${batch.map(() => "?").join(", ")})`
             : "";
         return {
-            sql: `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, error_code, abort_reason, dispatch_mode, terminal_reason, diag_stage, in_flight_since
+            sql: `SELECT id, thread_id, turn_id, status, started_at, heartbeat_at, completed_at, last_progress_at, error_code, abort_reason, dispatch_mode, terminal_reason, diag_stage, thread_projection_error_code, in_flight_since
             FROM agent_runs
             WHERE thread_id = ?${runIdFilter}${batch ? "" : " ORDER BY started_at DESC LIMIT ?"}`,
             args: [threadId, ...(batch ?? []), ...(batch ? [] : [limit])],
@@ -3524,6 +3731,7 @@ export async function listRunsForThread(threadId, options = {}) {
             dispatchMode: row.dispatch_mode ?? null,
             terminalReason: row.terminal_reason ?? null,
             diagStage: row.diag_stage ?? null,
+            threadProjectionErrorCode: row.thread_projection_error_code ?? null,
             inFlightSince: row.in_flight_since == null ? null : Number(row.in_flight_since),
         };
     });
@@ -3759,6 +3967,7 @@ export async function reapAllStaleRuns() {
     const scanned = await client.execute({
         sql: `SELECT id FROM agent_runs
           WHERE status = 'running'
+            AND ${earlierTurnProjectionPendingSql()}
             AND ${livenessBasisSql()} < ${backgroundAwareStaleCutoffSql()}
           ORDER BY started_at ASC
           LIMIT ${REAP_ALL_STALE_BATCH_LIMIT + 1}`,
@@ -4138,17 +4347,7 @@ async function finalizeStaleRun(runId, source, projectThread = true) {
     if (!projectThread)
         return true;
     try {
-        const { foldReapedRunIntoThread } = await import("./reaped-run-thread.js");
-        const disposition = await foldReapedRunIntoThread(runId);
-        if (disposition === "retry")
-            return false;
-        await getDbExec().execute({
-            sql: `UPDATE agent_runs
-            SET thread_projection_pending = FALSE
-            WHERE id = ? AND thread_projection_pending IS TRUE`,
-            args: [runId],
-        });
-        return true;
+        return (await projectRunThread(runId)) === "projected";
     }
     catch (error) {
         captureError(error, {
@@ -4162,6 +4361,122 @@ async function finalizeStaleRun(runId, source, projectThread = true) {
         return false;
     }
 }
+async function bumpRunThreadProjectionRetryOrder(runId) {
+    await getDbExec().execute({
+        sql: `UPDATE agent_runs
+          SET thread_projection_retry_order = nextval('agent_run_projection_retry_order_seq')
+          WHERE id = ? AND thread_projection_pending IS TRUE`,
+        args: [runId],
+    });
+}
+async function projectRunThread(runId) {
+    const { foldReapedRunIntoThread } = await import("./reaped-run-thread.js");
+    let disposition;
+    try {
+        disposition = await foldReapedRunIntoThread(runId);
+    }
+    catch (error) {
+        if (error instanceof AgentRunJournalUnreadableError) {
+            await getDbExec().execute({
+                sql: `UPDATE agent_runs
+              SET thread_projection_error_code = 'tool_call_journal_unreadable',
+                  thread_projection_retry_order = NULL
+              WHERE id = ? AND thread_projection_pending IS TRUE`,
+                args: [runId],
+            });
+            throw error;
+        }
+        try {
+            await bumpRunThreadProjectionRetryOrder(runId);
+        }
+        catch (retryOrderError) {
+            throw new AggregateError([error, retryOrderError], `Run ${runId} projection failed and its retry order could not be advanced`, { cause: error });
+        }
+        throw error;
+    }
+    if (disposition === "retry") {
+        await bumpRunThreadProjectionRetryOrder(runId);
+        return "deferred";
+    }
+    const client = getDbExec();
+    if (!client.transaction) {
+        throw new Error("Run thread projection completion requires transaction support");
+    }
+    await client.transaction(async (tx) => {
+        const { rows } = await tx.execute({
+            sql: `UPDATE agent_runs
+            SET thread_projection_pending = FALSE,
+                thread_projection_retry_order = NULL,
+                thread_projection_error_code = NULL
+            WHERE id = ? AND thread_projection_pending IS TRUE
+            RETURNING id`,
+            args: [runId],
+        });
+        if (rows.length === 0)
+            return;
+        // Give queued same-turn successors a dispatch window when their gate opens.
+        await tx.execute({
+            sql: `UPDATE agent_runs AS successor
+            SET heartbeat_at = ?
+            FROM agent_runs AS predecessor
+            WHERE predecessor.id = ?
+              AND successor.thread_id = predecessor.thread_id
+              AND COALESCE(successor.turn_id, successor.id) =
+                  COALESCE(predecessor.turn_id, predecessor.id)
+              AND successor.id <> predecessor.id
+              AND successor.status = 'running'
+              AND successor.dispatch_mode = 'background'
+              AND (
+                COALESCE(predecessor.continuation_order, 0) <
+                    COALESCE(successor.continuation_order, 0)
+                OR (
+                  COALESCE(predecessor.continuation_order, 0) =
+                      COALESCE(successor.continuation_order, 0)
+                  AND (
+                    predecessor.started_at < successor.started_at
+                    OR (
+                      predecessor.started_at = successor.started_at
+                      AND predecessor.id < successor.id
+                    )
+                  )
+                )
+              )`,
+            args: [Date.now(), runId],
+        });
+    });
+    return "projected";
+}
+/** Retry after repairing persisted source data; until success, the marker keeps later same-turn runs gated. */
+export async function retryRunThreadProjection(runId) {
+    await ensureRunTables();
+    const retried = await getDbExec().execute({
+        sql: `UPDATE agent_runs
+          SET thread_projection_error_code = NULL
+          WHERE id = ?
+            AND thread_projection_pending IS TRUE
+            AND ${earlierTurnProjectionPendingSql()}
+          RETURNING id`,
+        args: [runId],
+    });
+    if (retried.rows.length > 0) {
+        return { state: await projectRunThread(runId) };
+    }
+    const { rows } = await getDbExec().execute({
+        sql: `SELECT thread_projection_pending FROM agent_runs WHERE id = ?`,
+        args: [runId],
+    });
+    // Outcome pruning retains every run whose projection marker is still set.
+    if (rows.length === 0)
+        return { state: "untracked" };
+    const pending = rows[0]?.thread_projection_pending;
+    if (pending === true || pending === "t" || pending === 1) {
+        return { state: "deferred" };
+    }
+    if (pending === false || pending === "f" || pending === 0) {
+        return { state: "already_projected" };
+    }
+    throw new Error(`Run ${runId} has an unreadable thread projection marker`);
+}
 class AbortTerminalEventWriteError extends Error {
     runId;
     constructor(runId, cause) {
@@ -4309,9 +4624,13 @@ async function appendTerminalRunEventInTransaction(tx, runId, event, replaceExis
 }
 async function retryPendingStaleRunThreadProjections(client) {
     const { rows } = await client.execute({
-        sql: `SELECT id FROM agent_runs
+        sql: `SELECT id, thread_projection_error_code FROM agent_runs
           WHERE thread_projection_pending IS TRUE
-          ORDER BY completed_at ASC, id ASC
+            AND (thread_projection_error_code IS NOT NULL
+              OR ${earlierTurnProjectionPendingSql()})
+          ORDER BY CASE WHEN thread_projection_error_code IS NULL THEN 0 ELSE 1 END,
+                   thread_projection_retry_order ASC NULLS FIRST,
+                   completed_at ASC, id ASC
           LIMIT ${REAP_ALL_STALE_BATCH_LIMIT + 1}`,
         args: [],
     });
@@ -4319,7 +4638,12 @@ async function retryPendingStaleRunThreadProjections(client) {
     const pending = truncated ? rows.slice(0, REAP_ALL_STALE_BATCH_LIMIT) : rows;
     let failed = 0;
     for (const row of pending) {
-        const runId = row.id;
+        const projection = row;
+        if (projection.thread_projection_error_code != null) {
+            failed += 1;
+            continue;
+        }
+        const runId = projection.id;
         if (typeof runId !== "string")
             continue;
         if (!(await finalizeStaleRun(runId, "stale-projection-retry")))

package/dist/agent/tool-approval-store.d.ts

@@ -1,3 +1,4 @@
+import type { DbExec } from "../db/client.js";
 export { hashAgentToolApprovalKey } from "./approval-key.js";
 export declare const agentToolApprovalsStore: import("../db/store-registry.js").Store;
 export declare function ensureAgentToolApprovalTable(): Promise<void>;
@@ -27,6 +28,18 @@ export interface AgentToolApprovalPolicyBinding {
 export declare class AgentToolApprovalNotFoundError extends Error {
     constructor();
 }
+export declare function hasPendingAgentToolApproval(tx: Pick<DbExec, "execute">, input: {
+    ownerEmail: string;
+    orgId?: string | null;
+    threadId: string;
+}): Promise<boolean>;
+/** Denial is a durable resolution so queued turns can resume after a pause. */
+export declare function denyAgentToolApproval(input: {
+    approvalId: string;
+    ownerEmail: string;
+    orgId?: string | null;
+    threadId: string;
+}): Promise<"denied" | "already_resolved" | "not_applied">;
 /**
  * Saves "always allow" for one tool name. With `approvalId`, the grant takes
  * the revision of that approval row, the card the user clicked, so it never