@relaycast/engine 10.1.1 → 10.3.0

package/dist/engine/action.js

@@ -1,4 +1,4 @@
-import { and, asc, eq, inArray, isNotNull, isNull, lte, or, sql } from 'drizzle-orm';
+import { and, asc, eq, inArray, isNotNull, isNull, lt, lte, or, sql } from 'drizzle-orm';
 import { actions, actionInvocations, agents, agentNodeBindings, channelMembers, dmParticipants, nodes } from '../db/schema.js';
 import { waitForPendingInvocationRetry } from './invocationRetry.js';
 import { generateId } from './snowflake.js';
@@ -617,11 +617,13 @@ async function isNodeProviderLive(db, registry, workspaceId, nodeId, providerNam
     const provider = await getProvider(db, workspaceId, nodeId, providerName);
     return !!provider && isProviderLive(provider);
 }
-async function failInvocationForUnavailableProvider(db, workspaceId, invocationId) {
+async function failInvocationForUnavailableProvider(db, workspaceId, invocationId, expectedNodeLivenessRevision) {
     await db
         .update(actionInvocations)
         .set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])));
 }
 /**
  * Terminally fail one invocation ONLY while it is still open AND was never
@@ -648,7 +650,7 @@ async function failNeverDispatchedInvocation(db, workspaceId, invocationId, erro
  * enclosing transaction commits). A spawn invocation holding a native node
  * reservation releases that capacity, mirroring `completeNodeInvocation`.
  */
-async function failOpenInvocationRows(db, workspaceId, invocationIds, error) {
+async function failOpenInvocationRows(db, workspaceId, invocationIds, error, expectedNodeLivenessRevision) {
     if (invocationIds.length === 0)
         return [];
     // Capture reservation state before the update clears spawnReservedAt.
@@ -660,11 +662,15 @@ async function failOpenInvocationRows(db, workspaceId, invocationIds, error) {
         spawnReservedAt: actionInvocations.spawnReservedAt,
     })
         .from(actionInvocations)
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])));
     const failed = await db
         .update(actionInvocations)
         .set({ status: 'failed', error, completedAt: new Date(), spawnReservedAt: null })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), inArray(actionInvocations.id, invocationIds), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning();
     const failedIds = new Set(failed.map((row) => row.id));
     for (const row of held) {
@@ -732,7 +738,7 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
             if (options.failIfUnavailable === false) {
                 return { accepted: false, pending: false, providerUnavailable: true };
             }
-            await failInvocationForUnavailableProvider(args.db, args.workspaceId, args.invocationId);
+            await failInvocationForUnavailableProvider(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             throw codedError(`Provider "${args.providerName}" is offline for action "${args.action}"`, 'handler_unavailable', 503);
         }
         // The DB is the authoritative offline queue. Do not call sendToProvider
@@ -749,9 +755,10 @@ async function dispatchNodeProviderInvocation(args, options = {}) {
                 ? { id: expectedActionId, name: args.action }
                 : undefined,
             targetActionId: args.actionId ?? undefined,
+            expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
         });
         if (!accepted && args.invocationOrigin === 'registered_action') {
-            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             return { accepted: false, pending: false, settled };
         }
         return { accepted, pending: true };
@@ -1822,7 +1829,9 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts)
         ...(opts.targetActionId ? { actionId: opts.targetActionId } : {}),
         ...attemptFields,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedAction ? [
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+        : []), ...(opts.expectedAction ? [
         eq(actionInvocations.invocationOrigin, 'registered_action'),
         eq(actionInvocations.actionId, opts.expectedAction.id),
         sql `EXISTS (
@@ -1885,7 +1894,9 @@ async function claimRegisteredActionHandoff(db, args) {
         actionId: claimedActionId,
         ...attemptFields,
     })
-        .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), sql `NOT (
+        .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), eq(actionInvocations.actionId, args.expectedActionId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(args.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, args.expectedNodeLivenessRevision)]
+        : []), sql `NOT (
         ${actionInvocations.actionId} = ${claimedActionId}
         AND COALESCE(${actionInvocations.dispatchedNodeId}, '') = ${args.nodeId}
         AND COALESCE(${actionInvocations.dispatchedProvider}, ${DEFAULT_PROVIDER_NAME}) = ${args.providerName}
@@ -1922,7 +1933,7 @@ async function bindInvocationToRegisteredNodeAction(db, args) {
         .returning({ id: actionInvocations.id });
     return !!bound;
 }
-async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
+async function settleDeletedRegisteredAction(db, workspaceId, invocationId, expectedNodeLivenessRevision) {
     // A failed handoff can also mean another dispatcher already advanced the
     // same invocation to a different live action. Fail only the FK-null state
     // that proves pruning won; never clobber that concurrent dispatch winner.
@@ -1934,7 +1945,9 @@ async function settleDeletedRegisteredAction(db, workspaceId, invocationId) {
         completedAt: new Date(),
         spawnReservedAt: null,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.invocationOrigin, 'registered_action'), isNull(actionInvocations.actionId), or(isNull(actionInvocations.providerAcceptedAttempt), sql `${actionInvocations.providerAcceptedAttempt} <> ${actionInvocations.dispatchAttempts}`), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning();
     if (failed)
         return failed;
@@ -1952,13 +1965,15 @@ async function settleRegisteredActionSendFailure(db, args) {
         .returning();
     return failed ?? settleDeletedRegisteredAction(db, args.workspaceId, args.invocationId);
 }
-async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId) {
+async function snapshotInvocationHandlerNode(db, workspaceId, invocationId, nodeId, expectedNodeLivenessRevision) {
     const [updated] = await db
         .update(actionInvocations)
         .set({
         handlerNodeId: sql `COALESCE(${actionInvocations.handlerNodeId}, ${nodeId})`,
     })
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, expectedNodeLivenessRevision)]
+        : [])))
         .returning({ id: actionInvocations.id });
     return !!updated;
 }
@@ -2038,7 +2053,7 @@ async function dispatchNodeInvocation(args) {
     // owner performs the last-moment gate below immediately before the send.
     const snapshotted = args.agent
         ? true
-        : await snapshotInvocationHandlerNode(args.db, args.workspaceId, args.invocationId, args.nodeId);
+        : await snapshotInvocationHandlerNode(args.db, args.workspaceId, args.invocationId, args.nodeId, args.expectedNodeLivenessRevision);
     if (!snapshotted)
         return { accepted: false, pending: false, sent: false };
     const connectedBefore = args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName);
@@ -2061,10 +2076,11 @@ async function dispatchNodeInvocation(args) {
                 retryAfterAt: args.retryAfterAt,
                 reservationHeld: args.reservationHeld,
                 skipIncrementAttempts: args.skipIncrementAttempts,
+                expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
             })
             : null;
         if (!registeredNodeClaim) {
-            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+            const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
             return { accepted: false, pending: false, sent: false, settled };
         }
     }
@@ -2078,6 +2094,9 @@ async function dispatchNodeInvocation(args) {
             actionId: args.actionId,
             handlerAgentId: args.agent.id,
             recordAttempt: !args.skipIncrementAttempts,
+            ...(args.expectedNodeLivenessRevision !== undefined
+                ? { expectedNodeLivenessRevision: args.expectedNodeLivenessRevision }
+                : {}),
         }) ?? false)
         : registeredNodeClaim
             ? await (args.registry.sendAuthorizedActionToProvider?.(args.workspaceId, args.nodeId, args.providerName, frame, {
@@ -2128,12 +2147,22 @@ async function dispatchNodeInvocation(args) {
         }
         return { accepted: false, pending: false, sent: false };
     }
-    // Registered node-action state was committed by the pre-send handoff CAS.
-    // Pruning after provider acceptance may clear the action FK, but it cannot
-    // revoke work the provider already received or prevent route-owned completion.
+    // Registered node-action state was committed by the pre-send handoff CAS;
+    // agent-action state was committed by the socket owner's authorization CAS.
+    // Do not write either generation again after the frame is accepted.
     if (registeredNodeClaim) {
         return { accepted: true, pending, sent: true };
     }
+    if (args.agent && args.actionId) {
+        const [settled] = await args.db
+            .select()
+            .from(actionInvocations)
+            .where(and(eq(actionInvocations.workspaceId, args.workspaceId), eq(actionInvocations.id, args.invocationId)));
+        if (settled && !OPEN_INVOCATION_STATUSES.some((status) => status === settled.status)) {
+            return { accepted: false, pending: false, sent: true, settled };
+        }
+        return { accepted: true, pending, sent: true };
+    }
     const finalExpectedActionId = expectedActionId;
     const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
         pending,
@@ -2151,9 +2180,10 @@ async function dispatchNodeInvocation(args) {
         targetActionId: args.agent || args.actionId
             ? finalExpectedActionId ?? undefined
             : undefined,
+        expectedNodeLivenessRevision: args.expectedNodeLivenessRevision,
     });
     if (!accepted && args.invocationOrigin === 'registered_action') {
-        const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId);
+        const settled = await settleDeletedRegisteredAction(args.db, args.workspaceId, args.invocationId, args.expectedNodeLivenessRevision);
         return { accepted: false, pending: false, sent: true, settled };
     }
     if (!accepted && (guardedReleaseHash || guardedReleaseAgentId)) {
@@ -2351,7 +2381,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         return false;
     }
     if (isDeletedRegisteredActionInvocation(invocation)) {
-        await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted');
+        await failOpenInvocationRows(db, invocation.workspaceId, [invocation.id], 'action_deleted', opts.expectedNodeLivenessRevision);
         return false;
     }
     // Only release capacity the invocation actually holds. A shadowed spawn
@@ -2368,10 +2398,15 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         const capability = dispatchActionNameForInvocation(invocation.actionName, recordInput(invocation.input));
         const shadow = await fetchNodeAction(db, invocation.workspaceId, invocation.dispatchedNodeId, capability);
         if (shadow) {
-            await db
+            const [reset] = await db
                 .update(actionInvocations)
                 .set({ status: 'pending', dispatchedAt: null, spawnReservedAt: null, retryAfterAt: null })
-                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+                ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+                : [])))
+                .returning({ id: actionInvocations.id });
+            if (!reset)
+                return false;
             return false;
         }
     }
@@ -2391,6 +2426,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
             expectedActionId: invocation.actionId,
             invocationOrigin: invocation.invocationOrigin,
             retryAfterAt: opts.retryAfterAt ?? null,
+            expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
         });
         return dispatched.accepted;
     }
@@ -2439,6 +2475,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                         input,
                         queue: target.queue,
                         reservationHeld: false,
+                        expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                     }, { failIfUnavailable: false });
                     if (dispatched.providerUnavailable) {
                         providerRejectedNodeIds.add(placement.node.id);
@@ -2465,9 +2502,10 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                             ? { id: expectedActionId, name: actionToSend }
                             : undefined,
                         targetActionId: target?.id,
+                        expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                     });
                     if (!accepted && invocation.invocationOrigin === 'registered_action') {
-                        await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id);
+                        await settleDeletedRegisteredAction(db, invocation.workspaceId, invocation.id, opts.expectedNodeLivenessRevision);
                     }
                     return accepted;
                 }
@@ -2484,6 +2522,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
                     invocationOrigin: invocation.invocationOrigin,
                     input,
                     reservationHeld,
+                    expectedNodeLivenessRevision: opts.expectedNodeLivenessRevision,
                 });
                 return dispatched.accepted;
             }
@@ -2494,7 +2533,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         }
     }
     if (providerRejectedNodeIds.size > 0) {
-        await failInvocationForUnavailableProvider(db, invocation.workspaceId, invocation.id);
+        await failInvocationForUnavailableProvider(db, invocation.workspaceId, invocation.id, opts.expectedNodeLivenessRevision);
         return false;
     }
     await db
@@ -2507,18 +2546,31 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
         spawnReservedAt: null,
         retryAfterAt: opts.retryAfterAt ?? nextRetryAfter(invocation.dispatchAttempts + 1),
     })
-        .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.expectedNodeLivenessRevision !== undefined
+        ? [eq(actionInvocations.nodeLivenessRevision, opts.expectedNodeLivenessRevision)]
+        : [])));
     return false;
 }
-export async function rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId) {
+export async function rescheduleInvocationsForLostNode(db, registry, workspaceId, nodeId, opts = {}) {
     const rows = await db
         .select()
         .from(actionInvocations)
-        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), 
+    // The dispatch trigger stamps the accepting node generation. A reconnect
+    // can dispatch new work while settlement runs, including in the same
+    // timestamp second; only attempts older than this claim are recoverable.
+    ...(opts.claimRevision !== undefined
+        ? [lt(actionInvocations.nodeLivenessRevision, opts.claimRevision)]
+        : [])));
     let rescheduled = 0;
     for (const invocation of rows) {
         try {
-            if (await rescheduleNodeInvocation(db, registry, invocation, { retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1) })) {
+            if (await rescheduleNodeInvocation(db, registry, invocation, {
+                retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1),
+                ...(opts.claimRevision !== undefined
+                    ? { expectedNodeLivenessRevision: invocation.nodeLivenessRevision }
+                    : {}),
+            })) {
                 rescheduled++;
             }
         }
@@ -2534,7 +2586,9 @@ export async function rescheduleInvocationsForLostNode(db, registry, workspaceId
                 spawnReservedAt: null,
                 retryAfterAt: nextRetryAfter(invocation.dispatchAttempts + 1),
             })
-                .where(eq(actionInvocations.id, invocation.id));
+                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES), ...(opts.claimRevision !== undefined
+                ? [eq(actionInvocations.nodeLivenessRevision, invocation.nodeLivenessRevision)]
+                : [])));
         }
     }
     return rescheduled;