@relaycast/engine 8.2.1 → 8.2.2

package/dist/engine/action.js

@@ -4,12 +4,14 @@ import { generateId } from './snowflake.js';
 import { RELEASED_AGENT_STATUS, releasedAgentName } from './agent.js';
 import { randomHex, sha256Hex } from '../lib/crypto.js';
 import { codedError } from '../lib/httpError.js';
+import { D1_SAFE_IN_QUERY_CHUNK_SIZE } from '../lib/queryChunks.js';
 import { toFleetWireJson } from './deliveryWire.js';
 import { emitAgentExitedEffects, emitInvocationCompletionEffects, fleetInvocationId, } from './invocationCompletion.js';
 import { runAtomic, runAtomicWrites } from '../ports/database.js';
 import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
 import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
 const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
+const REPLAYED_INVOCATION = Symbol('replayed-action-invocation');
 export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
 /**
  * How long an agent handler's connection must be CONTINUOUSLY unreachable
@@ -20,6 +22,31 @@ export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
  * a signal instead of an unbounded hang.
  */
 export const ACTION_HANDLER_UNREACHABLE_TTL_MS = 120_000;
+/**
+ * Absolute age bound for a `pending` invocation that was NEVER dispatched to a
+ * node (`dispatch_attempts = 0`). The handler-unreachable TTL only fires once a
+ * dispatched handler connection has been observed offline; an invocation that
+ * never left the queue has no handler to observe, so the clock never starts
+ * and the row can sit `pending` forever. When the queue eventually drains
+ * (e.g. a node returns after a long outage), week-old spawn briefs come back
+ * to life as fresh agents that can evict the live resident under the same
+ * name — a delayed-action identity hazard, not inert backlog. Sized deliberately:
+ * long enough to ride out a weekend-scale node outage plus a day of buffer,
+ * far shorter than the observed multi-month backlog that motivated it.
+ */
+export const PENDING_INVOCATION_MAX_AGE_MS = 72 * 60 * 60 * 1000;
+/**
+ * Sleep between the never-dispatched sweep's candidate SELECT and its atomic
+ * UPDATE. `dispatchNodeInvocation` sends the frame before recording
+ * `dispatchAttempts += 1`, so a sweep landing between those two dispatcher
+ * calls could otherwise mark an in-flight dispatch `never_dispatched_expired`
+ * while the handler already has the frame. The atomic UPDATE re-checks
+ * `dispatch_attempts = 0`, so once the dispatcher's UPDATE lands, the sweep's
+ * WHERE excludes the row. This grace window (default 5s) covers the send→record
+ * gap by a wide margin — the dispatcher's UPDATE is one D1 round-trip — while
+ * still bounding sweep latency. Set to 0 in tests to keep them fast.
+ */
+export const NEVER_DISPATCHED_SWEEP_GRACE_MS = 5_000;
 const ACTION_RETRY_BACKOFF_MS = 5_000;
 const NODE_DRAIN_REQUEUE_RETRY_MS = 5_000;
 function capabilityName(capability) {
@@ -83,6 +110,19 @@ function recordInput(value) {
         ? value
         : {};
 }
+function canonicalJson(value) {
+    if (Array.isArray(value)) {
+        return `[${value.map((item) => canonicalJson(item)).join(',')}]`;
+    }
+    if (value && typeof value === 'object') {
+        const record = value;
+        return `{${Object.keys(record)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
+            .join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
 function publicAction(row) {
     return {
         id: row.id,
@@ -329,22 +369,128 @@ export async function deleteAction(db, workspaceId, name, options = {}) {
     return deleted;
 }
 async function createInvocation(db, workspaceId, action, data) {
-    const invocationId = `inv_${generateId()}`;
-    const [invocation] = await db
+    const invocationId = data.invocation_id ?? `inv_${generateId()}`;
+    const expectedActionName = action?.name ?? data.action_name ?? 'spawn';
+    const [created] = await db
         .insert(actionInvocations)
         .values({
         id: invocationId,
         workspaceId,
         actionId: action?.id ?? null,
-        actionName: action?.name ?? data.action_name ?? 'spawn',
+        actionName: expectedActionName,
         callerId: data.caller_id ?? null,
         callerName: data.caller_name ?? null,
+        handlerAgentId: data.handler_agent_id ?? action?.handlerAgentId ?? null,
+        handlerNodeId: data.handler_node_id ?? action?.handlerNodeId ?? null,
         input: data.input ?? {},
         status: data.status ?? 'pending',
     })
+        .onConflictDoNothing()
         .returning();
+    if (created)
+        return { invocation: created, replayed: false };
+    // Only a deterministic idempotency claim is expected to conflict. Its row
+    // is the durable pre-dispatch claim: a concurrent request or later retry
+    // observes the same invocation instead of sending another provider frame.
+    if (!data.invocation_id) {
+        throw new Error(`Action invocation id collision: ${invocationId}`);
+    }
+    const [existing] = await db
+        .select()
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
+    if (!existing) {
+        throw codedError('Idempotency claim could not be read after conflict', 'idempotency_unavailable', 503);
+    }
+    assertInvocationClaimMatches(existing, expectedActionName, data);
+    return { invocation: existing, replayed: true };
+}
+function assertInvocationClaimMatches(existing, expectedActionName, data) {
+    const samePayload = existing.actionName === expectedActionName
+        && existing.callerId === (data.caller_id ?? null)
+        && canonicalJson(recordInput(existing.input)) === canonicalJson(recordInput(data.input));
+    if (!samePayload) {
+        throw codedError('Idempotency-Key was reused with a different request payload', 'idempotency_key_reused', 409);
+    }
+}
+async function idempotentInvocationId(workspaceId, callerId, actionName, key) {
+    const digest = await sha256Hex(['action-invoke-v1', workspaceId, callerId, actionName, key].join('\0'));
+    return `inv_idem_${digest}`;
+}
+async function waitForInvocationReplayOutcome(db, workspaceId, invocation, options = {}) {
+    let current = invocation;
+    const deadline = Date.now() + 500;
+    while (!current.dispatchedNodeId
+        && !(options.acceptDispatchStarted && current.dispatchAttempts > 0)) {
+        if (!OPEN_INVOCATION_STATUSES.some((status) => status === current.status)) {
+            if (current.status === 'failed') {
+                throw codedError('Action invocation failed before provider dispatch completed', current.error ?? 'handler_unavailable', 503);
+            }
+            return current;
+        }
+        if (Date.now() >= deadline) {
+            // The winning request may have died after its durable claim but before
+            // provider dispatch started or completed. Fail retryably without
+            // releasing the key for another execution.
+            throw codedError('Idempotent action dispatch is still pending', 'idempotency_unavailable', 503);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+        const [updated] = await db
+            .select()
+            .from(actionInvocations)
+            .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocation.id)));
+        if (!updated) {
+            throw codedError('Idempotency claim disappeared during dispatch', 'idempotency_unavailable', 503);
+        }
+        current = updated;
+    }
+    return current;
+}
+/**
+ * Resolve a losing durable claim from the immutable invocation snapshots.
+ *
+ * This is shared by the optimistic pre-read and every branch-local insert
+ * conflict. The action row resolved by the losing request may have changed
+ * scope or handler after the winning claim was created, so it is never a safe
+ * source for replay classification.
+ */
+async function replayInvocationClaim(db, workspaceId, invocation) {
+    if (isSpawnInvocation(invocation.actionName)
+        || isReleaseInvocation(invocation.actionName)
+        || (!!invocation.handlerNodeId && !invocation.handlerAgentId)) {
+        return waitForInvocationReplayOutcome(db, workspaceId, invocation);
+    }
+    if (invocation.handlerAgentId) {
+        // A handler takeover can win after the durable claim insert but before the
+        // winner revalidates the handler and starts provider dispatch. Do not
+        // acknowledge that pre-send claim. Once the durable attempt marker exists,
+        // preserve the accepted post-send replay contract even if a later takeover
+        // terminally fails the row.
+        return waitForInvocationReplayOutcome(db, workspaceId, invocation, { acceptDispatchStarted: true });
+    }
     return invocation;
 }
+function invocationAck(invocation, { actionName = invocation.actionName, handlerAgentId = invocation.handlerAgentId, handlerNodeId = invocation.handlerNodeId, } = {}) {
+    return {
+        invocation_id: invocation.id,
+        action_name: actionName,
+        handler_agent_id: handlerAgentId,
+        // The snapshot is the handler returned by the original 201. The mutable
+        // dispatched node remains available separately for current lifecycle state.
+        handler_node_id: handlerNodeId ?? invocation.dispatchedNodeId,
+        dispatched_node_id: invocation.dispatchedNodeId,
+        input: recordInput(invocation.input),
+        status: invocation.status,
+        created_at: invocation.createdAt.toISOString(),
+    };
+}
+function markInvocationReplay(result) {
+    Object.defineProperty(result, REPLAYED_INVOCATION, { value: true });
+    return result;
+}
+export function wasInvocationReplayed(result) {
+    return REPLAYED_INVOCATION in result;
+}
 /**
  * Liveness rule for the connection that hosts an agent handler, mirroring the
  * drain-time rule: the provider socket must be connected, and when a provider
@@ -493,14 +639,20 @@ async function dispatchRelease(args) {
     if (!agent) {
         throw codedError(`Agent "${name}" not found`, 'agent_not_found', 404);
     }
-    const invocation = await createInvocation(args.db, args.workspaceId, null, {
+    const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
         input,
         caller_id: args.data.caller_id,
         caller_name: args.data.caller_name,
         action_name: 'release',
+        invocation_id: args.invocationId,
     });
+    if (replayed) {
+        const settled = await waitForInvocationReplayOutcome(args.db, args.workspaceId, invocation);
+        return markInvocationReplay(invocationAck(settled, { actionName: 'release' }));
+    }
     const completeLocally = async () => {
         const completedAt = new Date();
+        const exitNodeId = nodeId ?? agent.locationNodeId;
         // Keyed on the agent id, not on the clock: the id is already unique per
         // workspace, so the tombstone can never collide with an existing row (a
         // second release of the same row is idempotent). A timestamped name would
@@ -591,6 +743,9 @@ async function dispatchRelease(args) {
                 .update(actionInvocations)
                 .set({
                 status: 'completed',
+                // This is part of the same atomic unit as completion so a lost 201
+                // replays the exact handler node returned below.
+                handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${exitNodeId})`,
                 output: {
                     released: true,
                     // The roster row is retained as a tombstone so the agent's history
@@ -608,7 +763,6 @@ async function dispatchRelease(args) {
         const completed = results.at(-1);
         // External completion effects belong after the durable atomic unit: an
         // aborted local reap must never publish agent.exited.
-        const exitNodeId = nodeId ?? agent.locationNodeId;
         if (completed.length > 0 && args.completionDeps && exitNodeId) {
             await emitAgentExitedEffects(args.completionDeps, args.workspaceId, {
                 agentId: agent.id,
@@ -697,18 +851,44 @@ async function dispatchSpawn(args) {
     }
     const input = recordInput(args.data.input);
     const capability = dispatchActionNameForInvocation('spawn', input);
-    const invocation = await createInvocation(args.db, args.workspaceId, null, {
+    const { invocation, replayed } = await createInvocation(args.db, args.workspaceId, null, {
         input: args.data.input,
         caller_id: args.data.caller_id,
         caller_name: args.data.caller_name,
+        invocation_id: args.invocationId,
     });
-    const placement = await claimSpawnNode(args.db, args.workspaceId, {
-        actionName: 'spawn',
-        input,
-        callerId: args.data.caller_id,
-        preferredNodeId: args.targetNodeId,
-    });
+    if (replayed) {
+        const dispatched = await waitForInvocationReplayOutcome(args.db, args.workspaceId, invocation);
+        return markInvocationReplay(invocationAck(dispatched, { actionName: 'spawn' }));
+    }
+    let placement;
+    try {
+        placement = await claimSpawnNode(args.db, args.workspaceId, {
+            actionName: 'spawn',
+            input,
+            callerId: args.data.caller_id,
+            preferredNodeId: args.targetNodeId,
+        });
+    }
+    catch (error) {
+        // Placement is transactional and no provider frame can exist yet. Release
+        // only this untouched pre-placement claim so a keyed retry can try again
+        // after capacity recovers. Once any target/attempt state exists, the claim
+        // remains immutable and the normal replay path owns it.
+        await args.db
+            .delete(actionInvocations)
+            .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, invocation.id), eq(actionInvocations.status, 'pending'), isNull(actionInvocations.handlerNodeId), isNull(actionInvocations.dispatchedNodeId), isNull(actionInvocations.spawnReservedAt), eq(actionInvocations.dispatchAttempts, 0)));
+        throw error;
+    }
     const nodeId = placement.node.id;
+    // Placement and the durable idempotency claim are separate writes on D1.
+    // Publish the selected response target immediately; a concurrent replay in
+    // this narrow interval waits for this snapshot instead of returning null.
+    if (!await snapshotInvocationHandlerNode(args.db, args.workspaceId, invocation.id, nodeId)) {
+        if (!placement.queued)
+            await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
+        throw codedError('Spawn invocation is no longer available for placement', 'idempotency_unavailable', 503);
+    }
     // A registered `spawn:<harness>` action shadows native capacity on this node.
     // Capacity-direct delegation (ctx.spawnAgent) bypasses the shadow so a handler
     // that delegates cannot re-enter itself.
@@ -790,7 +970,7 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
             throw codedError(`${who} is not authorized to invoke action "${actionName}"`, 'action_denied', 403);
         }
     }
-    const invocation = await createInvocation(db, workspaceId, action, {
+    const { invocation } = await createInvocation(db, workspaceId, action, {
         input: data.input,
         caller_id: data.caller_id,
         caller_name: data.caller_name,
@@ -824,12 +1004,29 @@ export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data
 }
 export async function invokeAction(db, workspaceId, actionName, data, options = {}) {
     const action = await fetchAction(db, workspaceId, actionName, options.includeNodeScoped);
+    let invocationId;
+    if (options.idempotencyKey !== undefined) {
+        if (!data.caller_id) {
+            throw codedError('Authenticated caller is required for idempotent action invocation', 'idempotency_actor_required', 400);
+        }
+        invocationId = await idempotentInvocationId(workspaceId, data.caller_id, actionName, options.idempotencyKey);
+        const [existing] = await db
+            .select()
+            .from(actionInvocations)
+            .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId)));
+        if (existing) {
+            assertInvocationClaimMatches(existing, actionName, data);
+            const replay = await replayInvocationClaim(db, workspaceId, existing);
+            return markInvocationReplay(invocationAck(replay, { actionName }));
+        }
+    }
     if (!action && actionName === 'spawn') {
         return dispatchSpawn({
             db,
             registry: options.nodeConnections,
             workspaceId,
             data,
+            invocationId,
         });
     }
     if (!action && actionName === 'release') {
@@ -839,6 +1036,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
             completionDeps: options.completionDeps,
             workspaceId,
             data,
+            invocationId,
         });
     }
     if (!action) {
@@ -855,11 +1053,16 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
         if (!options.nodeConnections) {
             throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
         }
-        const invocation = await createInvocation(db, workspaceId, action, {
+        const { invocation, replayed } = await createInvocation(db, workspaceId, action, {
             input: data.input,
             caller_id: data.caller_id,
             caller_name: data.caller_name,
+            invocation_id: invocationId,
         });
+        if (replayed) {
+            const settled = await replayInvocationClaim(db, workspaceId, invocation);
+            return markInvocationReplay(invocationAck(settled, { actionName }));
+        }
         // Only mark the reservation held when we actually incremented the node's
         // reserved-capacity counter, so completion/reschedule release stays balanced.
         // If the reservation can't be taken (node offline / at capacity) the queued
@@ -924,11 +1127,21 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
             throw codedError(`Action "${actionName}" handler "${handlerAgent.name}" has no live connection`, 'handler_unavailable', 503);
         }
     }
-    const invocation = await createInvocation(db, workspaceId, action, {
+    const { invocation, replayed } = await createInvocation(db, workspaceId, action, {
         input: data.input,
         caller_id: data.caller_id,
         caller_name: data.caller_name,
+        handler_node_id: handlerAgent.locationNodeId,
+        invocation_id: invocationId,
     });
+    if (replayed) {
+        const settled = await replayInvocationClaim(db, workspaceId, invocation);
+        return markInvocationReplay(invocationAck(settled, { actionName }));
+    }
+    if (!options.nodeConnections.sendAuthorizedActionToProvider) {
+        await failOpenInvocationRows(db, workspaceId, [invocation.id], 'node_dispatch_unavailable');
+        throw codedError('Node adapter does not support owner-authorized agent action dispatch', 'node_dispatch_unavailable', 503);
+    }
     // Re-validate the handler pointer AFTER the insert. A takeover committing
     // between action resolution above and the insert misses this invocation in
     // its stranded snapshot (the row was not visible yet), which would leave it
@@ -955,7 +1168,14 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
         action: action.name,
         input: recordInput(invocation.input),
         agent: { id: handlerAgent.id, name: handlerAgent.name },
+        actionId: action.id,
     });
+    if (!dispatched.accepted) {
+        // A takeover can invalidate the last-moment adapter gate. Re-read the
+        // durable claim so the original request and its replay agree on the same
+        // pre-send failure instead of returning a pending 201.
+        await waitForInvocationReplayOutcome(db, workspaceId, invocation, { acceptDispatchStarted: true });
+    }
     return {
         invocation_id: invocation.id,
         action_name: actionName,
@@ -1107,6 +1327,9 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
         .update(actionInvocations)
         .set({
         ...stateFields,
+        // The first selected target is part of the immutable 201 response. Keep
+        // it even when this invocation is later rescheduled to a different node.
+        handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${nodeId})`,
         dispatchedNodeId: nodeId,
         dispatchedProvider: opts.providerName,
         spawnReservedAt: opts.reservationHeld ? new Date() : null,
@@ -1117,6 +1340,16 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
         .returning();
     return !!updated;
 }
+async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
+    const [updated] = await db
+        .update(actionInvocations)
+        .set({
+        handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${nodeId})`,
+    })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .returning({ id: actionInvocations.id });
+    return !!updated;
+}
 /**
  * Transition a drained offline-queue invocation into the live `dispatched` state
  * once its queued `action.invoke` frame is actually delivered on node
@@ -1144,8 +1377,24 @@ async function dispatchNodeInvocation(args) {
         ...(args.agent ? { agent_id: args.agent.id, agent_name: args.agent.name } : {}),
         input: toFleetWireJson(args.input),
     };
+    // Agent-hosted createInvocation() already persisted the immutable handler
+    // node. Avoid another awaited DB write after handler revalidation; the socket
+    // owner performs the last-moment gate below immediately before the send.
+    const snapshotted = args.agent
+        ? true
+        : await snapshotInvocationHandlerNode(args.db, args.workspaceId, args.invocationId, args.nodeId);
+    if (!snapshotted)
+        return { accepted: false, pending: false };
     const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
-    const sent = await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
+    const sent = args.agent && args.actionId
+        ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
+            kind: 'agent-action-v1',
+            invocationId: args.invocationId,
+            actionId: args.actionId,
+            handlerAgentId: args.agent.id,
+            recordAttempt: !args.skipIncrementAttempts,
+        }) ?? false)
+        : await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame);
     if (!sent)
         return { accepted: false, pending: false };
     const pending = !!args.pending || !connectedBefore;
@@ -1154,7 +1403,7 @@ async function dispatchNodeInvocation(args) {
         providerName: args.providerName,
         retryAfterAt: args.retryAfterAt,
         reservationHeld: args.reservationHeld,
-        skipIncrementAttempts: args.skipIncrementAttempts,
+        skipIncrementAttempts: args.skipIncrementAttempts || !!args.agent,
         actionId: args.actionId,
     });
     return { accepted, pending };
@@ -1185,6 +1434,7 @@ async function selectRetryPlacement(db, invocation, excludeNodeIds) {
 async function targetAgentForInvocation(db, invocation) {
     const [row] = await db
         .select({
+        actionId: actions.id,
         agentId: agents.id,
         agentName: agents.name,
         locationType: agents.locationType,
@@ -1197,13 +1447,20 @@ async function targetAgentForInvocation(db, invocation) {
         .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id)));
     if (!row || row.locationType !== 'via_node' || !row.nodeId)
         return null;
-    return { agentId: row.agentId, agentName: row.agentName, nodeId: row.nodeId, providerName: row.providerName };
+    return {
+        actionId: row.actionId,
+        agentId: row.agentId,
+        agentName: row.agentName,
+        nodeId: row.nodeId,
+        providerName: row.providerName,
+    };
 }
 export async function drainNodeInvocations(db, registry, workspaceId, nodeId, options = {}) {
     const now = new Date();
     const rows = await db
         .select({
         id: actionInvocations.id,
+        actionId: actions.id,
         workspaceId: actionInvocations.workspaceId,
         actionName: actionInvocations.actionName,
         input: actionInvocations.input,
@@ -1293,6 +1550,7 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
                 action: dispatchActionNameForInvocation(row.actionName, input),
                 input,
                 agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
+                actionId: targetAgent ? row.actionId ?? undefined : undefined,
                 retryAfterAt: new Date(Date.now() + ACTION_DISPATCH_TIMEOUT_MS),
                 reservationHeld,
                 skipIncrementAttempts: row.dispatchedNodeId === nodeId,
@@ -1347,6 +1605,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
             action: invocation.actionName,
             input: recordInput(invocation.input),
             agent: { id: targetAgent.agentId, name: targetAgent.agentName },
+            actionId: targetAgent.actionId,
             retryAfterAt: opts.retryAfterAt ?? null,
         });
         return dispatched.accepted;
@@ -1641,10 +1900,76 @@ async function failUnreachableAgentInvocations(db, registry, ttlMs, completionDe
         }
     }
 }
+/**
+ * Absolute-age bound for never-dispatched pending invocations. The
+ * handler-unreachable TTL keys off a dispatched connection going quiet; an
+ * invocation that never left the queue has no handler to observe, so it needs
+ * its own bound. Fails rows that are `pending`, have `dispatch_attempts = 0`,
+ * and are older than `maxAgeMs` with a distinguishing error so operators can
+ * tell a never-dispatched expiry from a handler-that-went-quiet expiry.
+ */
+async function failNeverDispatchedExpiredInvocations(db, maxAgeMs, graceMs, completionDeps) {
+    const cutoff = new Date(Date.now() - maxAgeMs);
+    const rows = await db
+        .select({ id: actionInvocations.id, workspaceId: actionInvocations.workspaceId })
+        .from(actionInvocations)
+        .where(and(eq(actionInvocations.status, 'pending'), eq(actionInvocations.dispatchAttempts, 0), lte(actionInvocations.createdAt, cutoff)));
+    if (rows.length === 0)
+        return;
+    // Give any concurrent dispatcher's send→record window (`dispatchNodeAttempt`
+    // UPDATE, one D1 round-trip) time to close before the atomic UPDATE fires.
+    // Combined with the UPDATE's `dispatch_attempts = 0` re-check, this makes an
+    // in-flight dispatch reliably invisible to the age sweep instead of racing
+    // with it. See `NEVER_DISPATCHED_SWEEP_GRACE_MS`.
+    if (graceMs > 0) {
+        await new Promise((resolve) => setTimeout(resolve, graceMs));
+    }
+    const byWorkspace = new Map();
+    for (const row of rows) {
+        const list = byWorkspace.get(row.workspaceId) ?? [];
+        list.push(row.id);
+        byWorkspace.set(row.workspaceId, list);
+    }
+    for (const [workspaceId, ids] of byWorkspace) {
+        // Chunk ids so the IN clause stays under D1's 100-bound-parameter cap. A per-
+        // chunk try/catch keeps one bad chunk from stalling the whole workspace, so a
+        // large stale backlog can actually drain across sweeps.
+        for (let i = 0; i < ids.length; i += D1_SAFE_IN_QUERY_CHUNK_SIZE) {
+            const chunk = ids.slice(i, i + D1_SAFE_IN_QUERY_CHUNK_SIZE);
+            try {
+                const failed = await failNeverDispatchedInvocationRows(db, workspaceId, chunk, cutoff, 'never_dispatched_expired');
+                if (completionDeps && failed.length > 0) {
+                    await emitFailedInvocationEffects(completionDeps, workspaceId, failed);
+                }
+            }
+            catch {
+                // Leave this chunk for the next sweep; other chunks still get their shot.
+            }
+        }
+    }
+}
+/**
+ * Terminally fail never-dispatched rows atomically: the UPDATE re-checks the
+ * SELECT predicates (`status = 'pending'`, `dispatch_attempts = 0`, `created_at
+ * <= cutoff`), so a row that was concurrently dispatched between the sweep's
+ * SELECT and this UPDATE is skipped instead of being killed mid-flight. Never-
+ * dispatched rows carry no spawn reservation (dispatch_attempts = 0 means no
+ * node was ever chosen), so no capacity release is needed.
+ */
+async function failNeverDispatchedInvocationRows(db, workspaceId, invocationIds, cutoff, error) {
+    if (invocationIds.length === 0)
+        return [];
+    return await db
+        .update(actionInvocations)
+        .set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), eq(actionInvocations.status, 'pending'), eq(actionInvocations.dispatchAttempts, 0), lte(actionInvocations.createdAt, cutoff)))
+        .returning();
+}
 export async function sweepTimedOutInvocations(db, registry, opts = {}) {
     const sweepOpts = typeof opts === 'number' || opts === null ? {} : opts;
     const timeoutMs = typeof opts === 'number' ? opts : sweepOpts.timeoutMs ?? ACTION_DISPATCH_TIMEOUT_MS;
     await failUnreachableAgentInvocations(db, registry, sweepOpts.handlerUnreachableTtlMs ?? ACTION_HANDLER_UNREACHABLE_TTL_MS, sweepOpts.completionDeps);
+    await failNeverDispatchedExpiredInvocations(db, sweepOpts.pendingInvocationMaxAgeMs ?? PENDING_INVOCATION_MAX_AGE_MS, sweepOpts.neverDispatchedSweepGraceMs ?? NEVER_DISPATCHED_SWEEP_GRACE_MS, sweepOpts.completionDeps);
     const now = new Date();
     const cutoff = new Date(Date.now() - timeoutMs);
     const rows = await db