@relaycast/engine 10.2.0 → 10.3.1

package/dist/engine/action.js

@@ -1,4 +1,4 @@
-import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
+import { and, asc, eq, inArray, isNotNull, isNull, like, lt, lte, or, sql } from 'drizzle-orm';
 import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
 import { waitForPendingInvocationRetry } from './invocationRetry.js';
 import { generateId } from './snowflake.js';
@@ -617,11 +617,13 @@ async function isNodeProviderLive(db, registry, workspaceId, nodeId, providerNam
     const provider = await getProvider(db, workspaceId, nodeId, providerName);
     return !!provider && isProviderLive(provider);
 }
-async function failInvocationForUnavailableProvider(db, workspaceId, invocationId) {
+async function failInvocationForUnavailableProvider(db, workspaceId, invocationId, expectedNodeLivenessRevision) {
     await db
         .update(actionInvocations)
         .set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])));
 }
 /**
  * Terminally fail one invocation ONLY while it is still open AND was never
@@ -648,7 +650,7 @@ async function failNeverDispatchedInvocation(db, workspaceId, invocationId, erro
  * enclosing transaction commits). A spawn invocation holding a native node
  * reservation releases that capacity, mirroring `completeNodeInvocation`.
  */
-async function failOpenInvocationRows(db, workspaceId, invocationIds, error) {
+async function failOpenInvocationRows(db, workspaceId, invocationIds, error, expectedNodeLivenessRevision) {
     if (invocationIds.length === 0)
         return [];
     // Capture reservation state before the update clears spawnReservedAt.
@@ -660,11 +662,15 @@ async function failOpenInvocationRows(db, workspaceId, invocationIds, error) {
         spawnReservedAt: actionInvocations.spawnReservedAt,
     })
         .from(actionInvocations)
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])));
     const failed = await db
         .update(actionInvocations)
         .set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning();
     const failedIds = new Set(failed.map((row) => row.id));
     for (const row of held) {
@@ -732,7 +738,7 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
             if (options.failIfUnavailable === false) {
                 return { accepted: false, pending: false, providerUnavailable: true };
             }
-            await failInvocationForUnavailableProvider(args.db, args.workspaceId, args.invocationId);
+            await failInvocationForUnavailableProvider(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             throw codedError(`Provider "${args.providerName}" is offline for action "${args.action}"`, 'handler_unavailable', 503);
         }
         // The DB is the authoritative offline queue. Do not call sendToProvider
@@ -749,9 +755,10 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
                 ? { id: expectedActionId, name: args.action }
                 : undefined,
             targetActionId: args.actionId ?? undefined,
+            expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
         });
         if (!accepted && args.invocationOrigin === 'registered_action') {
-            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             return { accepted: false, pending: false, settled };
         }
         return { accepted, pending: true };
@@ -1258,6 +1265,16 @@ async function dispatchSpawn(args) {
         pending: placement.queued,
         reservationHeld: !placement.queued,
     });
+    // Placement reserved capacity for an attempt that never landed. Return it
+    // first, then mark the row unsent: only the mark makes it drainable, so a
+    // drain never sees it while this send is in flight or still holding the
+    // reservation.
+    if (!dispatched.accepted && !placement.queued) {
+        await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
+    }
+    if (!dispatched.sent) {
+        await markUnsentNativeSpawn(args.db, args.workspaceId, invocation.id, nodeId);
+    }
     return spawnResult(invocation, nodeId, dispatched);
 }
 /**
@@ -1822,7 +1839,9 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
         ...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
         ...attemptFields,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+        : []), ...(opts.expectedAction ? [
         eq(actionInvocations.invocationOrigin, 'registered_action'),
         eq(actionInvocations.actionId, opts.expectedAction.id),
         sql `EXISTS (
@@ -1885,7 +1904,9 @@ async function claimRegisteredActionHandoff(db, args) {
         actionId: claimedActionId,
         ...attemptFields,
     })
-        .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
+        .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(args.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, args.expectedNodeLivenessRevision)]
+        : []), sql `NOT (
         ${actionInvocations.actionId} = ${claimedActionId}
         AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
         AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
@@ -1922,7 +1943,7 @@ async function bindInvocationToRegisteredNodeAction(db, args) {
         .returning({ id: actionInvocations.id });
     return !!bound;
 }
-async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
+async function settleDeletedRegisteredAction(db, workspaceId, invocationId, expectedNodeLivenessRevision) {
     // A failed handoff can also mean another dispatcher already advanced the
     // same invocation to a different live action. Fail only the FK-null state
     // that proves pruning won; never clobber that concurrent dispatch winner.
@@ -1934,7 +1955,9 @@ async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
         completedAt: new Date(),
         spawnReservedAt: null,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning();
     if (failed)
         return failed;
@@ -1952,13 +1975,15 @@ async function settleRegisteredActionSendFailure(db, args) {
         .returning();
     return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
 }
-async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
+async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId, expectedNodeLivenessRevision) {
     const [updated] = await db
         .update(actionInvocations)
         .set({
         handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${nodeId})`,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning({ id: actionInvocations.id });
     return !!updated;
 }
@@ -2038,7 +2063,7 @@ async function dispatchNodeInvocation(args) {
     // owner performs the last-moment gate below immediately before the send.
     const snapshotted = args.agent
         ? true
-        : await snapshotInvocationHandlerNode(args.db, args.workspaceId, args.invocationId, args.nodeId);
+        : await snapshotInvocationHandlerNode(args.db, args.workspaceId, args.invocationId, args.nodeId, args.expectedNodeLivenessRevision);
     if (!snapshotted)
         return { accepted: false, pending: false, sent: false };
     const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
@@ -2061,10 +2086,11 @@ async function dispatchNodeInvocation(args) {
                 retryAfterAt: args.retryAfterAt,
                 reservationHeld: args.reservationHeld,
                 skipIncrementAttempts: args.skipIncrementAttempts,
+                expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
             })
             : null;
         if (!registeredNodeClaim) {
-            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             return { accepted: false, pending: false, sent: false, settled };
         }
     }
@@ -2078,6 +2104,9 @@ async function dispatchNodeInvocation(args) {
             actionId: args.actionId,
             handlerAgentId: args.agent.id,
             recordAttempt: !args.skipIncrementAttempts,
+            ...(args.expectedNodeLivenessRevision !== undefined
+                ? { expectedNodeLivenessRevision: args.expectedNodeLivenessRevision }
+                : {}),
         }) ?? false)
         : registeredNodeClaim
             ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
@@ -2128,12 +2157,22 @@ async function dispatchNodeInvocation(args) {
         }
         return { accepted: false, pending: false, sent: false };
     }
-    // Registered node-action state was committed by the pre-send handoff CAS.
-    // Pruning after provider acceptance may clear the action FK, but it cannot
-    // revoke work the provider already received or prevent route-owned completion.
+    // Registered node-action state was committed by the pre-send handoff CAS;
+    // agent-action state was committed by the socket owner's authorization CAS.
+    // Do not write either generation again after the frame is accepted.
     if (registeredNodeClaim) {
         return { accepted: true, pending, sent: true };
     }
+    if (args.agent && args.actionId) {
+        const [settled] = await args.db
+            .select()
+            .from(actionInvocations)
+            .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
+        if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
+            return { accepted: false, pending: false, sent: true, settled };
+        }
+        return { accepted: true, pending, sent: true };
+    }
     const finalExpectedActionId = expectedActionId;
     const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
         pending,
@@ -2151,9 +2190,10 @@ async function dispatchNodeInvocation(args) {
         targetActionId: args.agent || args.actionId
             ? finalExpectedActionId ?? undefined
             : undefined,
+        expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
     });
     if (!accepted && args.invocationOrigin === 'registered_action') {
-        const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+        const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
         return { accepted: false, pending: false, sent: true, settled };
     }
     if (!accepted && (guardedReleaseHash || guardedReleaseAgentId)) {
@@ -2214,6 +2254,53 @@ async function targetAgentForInvocation(db, invocation) {
         providerName: row.providerName,
     };
 }
+/**
+ * A native spawn whose first `action.invoke` send failed. A send can fail on a
+ * live node — the provider socket is between reconnects, or the node's owner
+ * rejects the send under pressure — and Cloud keeps no per-node frame queue, so
+ * such a row records no dispatch attempt and no `dispatchedNodeId`. Its
+ * placement survives only in the immutable `handlerNodeId` snapshot.
+ *
+ * `handlerUnreachableSince` marks the send as settled-unsent. The same
+ * snapshot exists while a first send is still in flight, so the mark, not the
+ * snapshot, is what lets a drain re-send. Built-in spawns have no action row,
+ * so the agent-handler reachability sweep (which joins on one) never reads it.
+ */
+function unsentNativeSpawnOn(workspaceId, invocationId, nodeId) {
+    return and(eq(actionInvocations.workspaceId, workspaceId), ...(invocationId ? [eq(actionInvocations.id, invocationId)] : []), eq(actionInvocations.status, 'pending'), isNull(actionInvocations.actionId), isNull(actionInvocations.dispatchedNodeId), eq(actionInvocations.dispatchAttempts, 0), ...(nodeId ? [eq(actionInvocations.handlerNodeId, nodeId)] : []), or(eq(actionInvocations.actionName, 'spawn'), like(actionInvocations.actionName, 'spawn:%')));
+}
+async function markUnsentNativeSpawn(db, workspaceId, invocationId, nodeId) {
+    await db
+        .update(actionInvocations)
+        .set({ handlerUnreachableSince: new Date() })
+        .where(unsentNativeSpawnOn(workspaceId, invocationId, nodeId));
+}
+/** One drain wins the re-send; a loser, or a send still in flight, sees no mark. */
+async function claimUnsentNativeSpawn(db, workspaceId, invocationId) {
+    const [claimed] = await db
+        .update(actionInvocations)
+        .set({ handlerUnreachableSince: null })
+        .where(and(unsentNativeSpawnOn(workspaceId, invocationId, null), isNotNull(actionInvocations.handlerUnreachableSince)))
+        .returning({ id: actionInvocations.id });
+    return !!claimed;
+}
+/** `registry`, calling `onSend` before any provider send starts. */
+function observeSends(registry, onSend) {
+    return new Proxy(registry, {
+        get(target, key) {
+            const value = Reflect.get(target, key, target);
+            if (typeof value !== 'function')
+                return value;
+            if (key === 'sendToProvider' || key === 'sendAuthorizedActionToProvider') {
+                return (...args) => {
+                    onSend();
+                    return Reflect.apply(value, target, args);
+                };
+            }
+            return value.bind(target);
+        },
+    });
+}
 export async function drainNodeInvocations(db, registry, workspaceId, nodeId, options = {}) {
     const now = new Date();
     const rows = await db
@@ -2229,6 +2316,7 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
         spawnReservedAt: actionInvocations.spawnReservedAt,
         dispatchedNodeId: actionInvocations.dispatchedNodeId,
         dispatchedProvider: actionInvocations.dispatchedProvider,
+        placedNodeId: actionInvocations.handlerNodeId,
         handlerNodeId: actions.handlerNodeId,
         handlerProvider: actions.handlerProvider,
         handlerAgentId: agents.id,
@@ -2242,7 +2330,10 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
         .leftJoin(agents, eq(actions.handlerAgentId, agents.id))
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.status, 'pending'), options.includeDeferred
         ? undefined
-        : or(isNull(actionInvocations.retryAfterAt), lte(actionInvocations.retryAfterAt, now)), or(eq(actionInvocations.dispatchedNodeId, nodeId), eq(actions.handlerNodeId, nodeId), and(eq(agents.locationType, 'via_node'), eq(agents.locationNodeId, nodeId)))))
+        : or(isNull(actionInvocations.retryAfterAt), lte(actionInvocations.retryAfterAt, now)), or(eq(actionInvocations.dispatchedNodeId, nodeId), eq(actions.handlerNodeId, nodeId), and(eq(agents.locationType, 'via_node'), eq(agents.locationNodeId, nodeId)), 
+    // A native spawn whose first send failed on this node: no attempt
+    // landed, so no other column names it.
+    and(unsentNativeSpawnOn(workspaceId, null, nodeId), isNotNull(actionInvocations.handlerUnreachableSince)))))
         .orderBy(asc(actionInvocations.createdAt));
     let drained = 0;
     for (const row of rows) {
@@ -2256,7 +2347,13 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
             && row.handlerAgentNodeId
             ? { id: row.handlerAgentId, name: row.handlerAgentName, nodeId: row.handlerAgentNodeId }
             : null;
-        const targetNodeId = targetAgent?.nodeId ?? row.handlerNodeId ?? row.dispatchedNodeId;
+        // Only the unsent-spawn branch above matches a row with no action and no
+        // dispatched node; its placed node is the target.
+        const unsentSpawn = row.actionId === null && row.dispatchedNodeId === null;
+        const targetNodeId = targetAgent?.nodeId
+            ?? row.handlerNodeId
+            ?? row.dispatchedNodeId
+            ?? (unsentSpawn ? row.placedNodeId : null);
         if (targetNodeId !== nodeId)
             continue;
         const input = recordInput(row.input);
@@ -2284,12 +2381,24 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
         // node, for which connection state remains the available liveness signal.
         if (!(await isHandlerConnectionLive(db, registry, workspaceId, nodeId, providerName)))
             continue;
+        // Claim before reserving, so concurrent drains cannot both reserve and send.
+        if (unsentSpawn && !await claimUnsentNativeSpawn(db, workspaceId, row.id))
+            continue;
+        // A definitely-unsent outcome below restores the mark, after returning
+        // any reservation this drain took. Whether a send started separates a
+        // pre-send throw (unsent) from a throw with an unknown outcome.
+        const remarkUnsent = () => unsentSpawn
+            ? markUnsentNativeSpawn(db, workspaceId, row.id, nodeId)
+            : Promise.resolve();
+        let sendStarted = false;
+        const sendRegistry = unsentSpawn ? observeSends(registry, () => { sendStarted = true; }) : registry;
         let reservedForDrain = false;
         try {
             let reservationHeld = nativeSpawn && !!row.spawnReservedAt;
             if (nativeSpawn && !reservationHeld) {
                 const reserved = await reserveNodeCapacity(db, workspaceId, nodeId);
                 if (!reserved) {
+                    await remarkUnsent();
                     const [node] = await db
                         .select({
                         status: nodes.status,
@@ -2311,7 +2420,7 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
             }
             const dispatched = await dispatchNodeInvocation({
                 db,
-                registry,
+                registry: sendRegistry,
                 workspaceId,
                 invocationId: row.id,
                 nodeId,
@@ -2331,13 +2440,21 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId, op
             }
             else if (reservedForDrain) {
                 await releaseNodeCapacity(db, workspaceId, nodeId);
+                reservedForDrain = false;
             }
+            if (!dispatched.sent)
+                await remarkUnsent();
         }
         catch {
             if (reservedForDrain) {
                 await releaseNodeCapacity(db, workspaceId, nodeId).catch(() => { });
             }
             // Leave the invocation pending; a later drain or timeout sweep can retry.
+            // A throw before any send leaves the spawn unsent, so it is re-marked. A
+            // throw once a send started is an unknown outcome, so the spawn stays
+            // unmarked and only the never-dispatched sweep ends it.
+            if (!sendStarted)
+                await remarkUnsent().catch(() => { });
         }
     }
     return drained;
@@ -2351,7 +2468,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         return false;
     }
     if (isDeletedRegisteredActionInvocation(invocation)) {
-        await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
+        await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted', opts.expectedNodeLivenessRevision);
         return false;
     }
     // Only release capacity the invocation actually holds. A shadowed spawn
@@ -2368,10 +2485,15 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         const capability = dispatchActionNameForInvocation(invocation.actionName, recordInput(invocation.input));
         const shadow = await fetchNodeAction(db, invocation.workspaceId, invocation.dispatchedNodeId, capability);
         if (shadow) {
-            await db
+            const [reset] = await db
                 .update(actionInvocations)
                 .set({ status: 'pending', dispatchedAt: null, spawnReservedAt: null, retryAfterAt: null })
-                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+                ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+                : [])))
+                .returning({ id: actionInvocations.id });
+            if (!reset)
+                return false;
             return false;
         }
     }
@@ -2391,6 +2513,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
             expectedActionId: invocation.actionId,
             invocationOrigin: invocation.invocationOrigin,
             retryAfterAt: opts.retryAfterAt ?? null,
+            expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
         });
         return dispatched.accepted;
     }
@@ -2439,6 +2562,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                         input,
                         queue: target.queue,
                         reservationHeld: false,
+                        expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                     }, { failIfUnavailable: false });
                     if (dispatched.providerUnavailable) {
                         providerRejectedNodeIds.add(placement.node.id);
@@ -2465,9 +2589,10 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                             ? { id: expectedActionId, name: actionToSend }
                             : undefined,
                         targetActionId: target?.id,
+                        expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                     });
                     if (!accepted && invocation.invocationOrigin === 'registered_action') {
-                        await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
+                        await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id, opts.expectedNodeLivenessRevision);
                     }
                     return accepted;
                 }
@@ -2484,6 +2609,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                     invocationOrigin: invocation.invocationOrigin,
                     input,
                     reservationHeld,
+                    expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                 });
                 return dispatched.accepted;
             }
@@ -2494,7 +2620,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         }
     }
     if (providerRejectedNodeIds.size > 0) {
-        await failInvocationForUnavailableProvider(db, invocation.workspaceId, invocation.id);
+        await failInvocationForUnavailableProvider(db, invocation.workspaceId, invocation.id, opts.expectedNodeLivenessRevision);
         return false;
     }
     await db
@@ -2507,18 +2633,31 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         spawnReservedAt: null,
         retryAfterAt: opts.retryAfterAt ?? nextRetryAfter(invocation.dispatchAttempts + 1),
     })
-        .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+        : [])));
     return false;
 }
-export async function rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId) {
+export async function rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId, opts = {}) {
     const rows = await db
         .select()
         .from(actionInvocations)
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), 
+    // The dispatch trigger stamps the accepting node generation. A reconnect
+    // can dispatch new work while settlement runs, including in the same
+    // timestamp second; only attempts older than this claim are recoverable.
+    ...(opts.claimRevision !== undefined
+        ? [lt(actionInvocations.nodeLivenessRevision, opts.claimRevision)]
+        : [])));
     let rescheduled = 0;
     for (const invocation of rows) {
         try {
-            if (await rescheduleNodeInvocation(db, registry, invocation, { retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1) })) {
+            if (await rescheduleNodeInvocation(db, registry, invocation, {
+                retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1),
+                ...(opts.claimRevision !== undefined
+                    ? { expectedNodeLivenessRevision: invocation.nodeLivenessRevision }
+                    : {}),
+            })) {
                 rescheduled++;
             }
         }
@@ -2534,7 +2673,9 @@ export async function rescheduleInvocationsForLostNode(db, registry, workspaceId
                 spawnReservedAt: null,
                 retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1),
             })
-                .where(eq(actionInvocations.id, invocation.id));
+                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.claimRevision !== undefined
+                ? [eq(actionInvocations.nodeLivenessRevision, invocation.nodeLivenessRevision)]
+                : [])));
         }
     }
     return rescheduled;