@relaycast/engine 5.1.1 → 6.0.1

package/dist/engine/action.js

@@ -4,6 +4,7 @@ import { generateId } from './snowflake.js';
 import { codedError } from '../lib/httpError.js';
 import { toFleetWireJson } from './deliveryWire.js';
 import { claimSpawnNode, chooseNodeForAction, isNodeLive, releaseNodeCapacity, reserveNodeCapacity } from './placement.js';
+import { DEFAULT_PROVIDER_NAME, capacityProviderName, getProvider, isProviderLive } from './nodeProvider.js';
 const OPEN_INVOCATION_STATUSES = ['pending', 'dispatched', 'invoked'];
 export const ACTION_DISPATCH_TIMEOUT_MS = 30_000;
 const ACTION_RETRY_BACKOFF_MS = 5_000;
@@ -84,11 +85,39 @@ function publicAction(row) {
         created_at: row.createdAt.toISOString(),
     };
 }
-async function fetchAction(db, workspaceId, actionName) {
+/** Resolution priority when a name has several handlers: agent-hosted, then a
+ * node action with a workspace-global alias, then a plain node-scoped action. */
+function actionResolutionRank(row) {
+    if (row.handlerNodeId === null)
+        return 0;
+    if (row.isGlobal)
+        return 1;
+    return 2;
+}
+/**
+ * Resolve an action by name for invocation. The workspace-scoped invoke
+ * (`POST /v1/actions/:name/invoke`) reaches only agent-hosted actions and node
+ * actions that claimed a workspace-global alias — plain node-scoped actions are
+ * node-addressed there. `includeNodeScoped` lifts that filter for callers that
+ * bind to a name without a node (message triggers), so a trigger can fire a
+ * fleet-provider action; the resolved node-scoped row is then dispatched
+ * node-addressed by {@link invokeAction}. `(workspaceId, name)` is not unique
+ * across nodes, so a stable id tiebreak keeps resolution deterministic.
+ */
+async function fetchAction(db, workspaceId, actionName, includeNodeScoped = false) {
+    const rows = await db
+        .select()
+        .from(actions)
+        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, actionName), eq(actions.isActive, true), ...(includeNodeScoped ? [] : [or(isNull(actions.handlerNodeId), eq(actions.isGlobal, true))])));
+    return rows.sort((a, b) => actionResolutionRank(a) - actionResolutionRank(b) ||
+        (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))[0] ?? null;
+}
+/** Resolve a node-addressed action by its unique `(node, name)` key. */
+async function fetchNodeAction(db, workspaceId, nodeId, actionName) {
     const [action] = await db
         .select()
         .from(actions)
-        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.name, actionName), eq(actions.isActive, true)));
+        .where(and(eq(actions.workspaceId, workspaceId), eq(actions.handlerNodeId, nodeId), eq(actions.name, actionName), eq(actions.isActive, true)));
     return action ?? null;
 }
 export async function registerAction(db, workspaceId, data) {
@@ -224,6 +253,52 @@ async function createInvocation(db, workspaceId, action, data) {
         .returning();
     return invocation;
 }
+/** Is a node provider currently invokable — connection up AND heartbeat handlers_live. */
+async function isNodeProviderLive(db, registry, workspaceId, nodeId, providerName) {
+    if (!registry.isProviderConnected(workspaceId, nodeId, providerName))
+        return false;
+    const provider = await getProvider(db, workspaceId, nodeId, providerName);
+    return !!provider && isProviderLive(provider);
+}
+async function failInvocationForUnavailableProvider(db, workspaceId, invocationId) {
+    await db
+        .update(actionInvocations)
+        .set({ status: 'failed', error: 'handler_unavailable', completedAt: new Date(), spawnReservedAt: null })
+        .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+}
+/**
+ * Dispatch an invoke to a specific provider on a node. When the provider is
+ * offline the invoke fails fast (the invocation is failed) unless it opted into
+ * the per-provider offline queue. Retry placement can request a non-terminal
+ * unavailable result so it can try another node before failing the invocation.
+ */
+async function dispatchNodeProviderInvocation(args, options = {}) {
+    const live = await isNodeProviderLive(args.db, args.registry, args.workspaceId, args.nodeId, args.providerName);
+    if (!live) {
+        if (!args.queue) {
+            // Release any capacity reserved for this fail-fast spawn so the node's
+            // reserved-agent count doesn't stay inflated.
+            if (args.reservationHeld) {
+                await releaseNodeCapacity(args.db, args.workspaceId, args.nodeId);
+            }
+            if (options.failIfUnavailable === false) {
+                return { accepted: false, pending: false, providerUnavailable: true };
+            }
+            await failInvocationForUnavailableProvider(args.db, args.workspaceId, args.invocationId);
+            throw codedError(`Provider "${args.providerName}" is offline for action "${args.action}"`, 'handler_unavailable', 503);
+        }
+        // The DB is the authoritative offline queue. Do not call sendToProvider
+        // here: a connected provider whose heartbeat says handlers_live=false still
+        // has a socket, and sendToProvider would deliver the frame immediately.
+        const accepted = await dispatchNodeAttempt(args.db, args.workspaceId, args.invocationId, args.nodeId, {
+            pending: true,
+            reservationHeld: args.reservationHeld,
+            actionId: args.actionId,
+        });
+        return { accepted, pending: true };
+    }
+    return dispatchNodeInvocation({ ...args });
+}
 async function dispatchRelease(args) {
     if (!args.registry) {
         throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
@@ -249,12 +324,14 @@ async function dispatchRelease(args) {
         caller_name: args.data.caller_name,
         action_name: 'release',
     });
+    // Release is a capacity operation handled by the provider hosting the agent.
     const dispatched = await dispatchNodeInvocation({
         db: args.db,
         registry: args.registry,
         workspaceId: args.workspaceId,
         invocationId: invocation.id,
         nodeId: agent.locationNodeId,
+        providerName: agent.providerName,
         action: 'release',
         input,
     });
@@ -269,10 +346,24 @@ async function dispatchRelease(args) {
         created_at: invocation.createdAt.toISOString(),
     };
 }
+function spawnResult(invocation, nodeId, dispatched) {
+    return {
+        invocation_id: invocation.id,
+        action_name: 'spawn',
+        handler_agent_id: null,
+        handler_node_id: nodeId,
+        dispatched_node_id: dispatched.accepted ? nodeId : null,
+        input: recordInput(invocation.input),
+        status: dispatched.accepted ? (dispatched.pending ? 'pending' : 'dispatched') : 'pending',
+        created_at: invocation.createdAt.toISOString(),
+    };
+}
 async function dispatchSpawn(args) {
     if (!args.registry) {
         throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
     }
+    const input = recordInput(args.data.input);
+    const capability = dispatchActionNameForInvocation('spawn', input);
     const invocation = await createInvocation(args.db, args.workspaceId, null, {
         input: args.data.input,
         caller_id: args.data.caller_id,
@@ -280,33 +371,126 @@ async function dispatchSpawn(args) {
     });
     const placement = await claimSpawnNode(args.db, args.workspaceId, {
         actionName: 'spawn',
-        input: args.data.input,
+        input,
         callerId: args.data.caller_id,
+        preferredNodeId: args.targetNodeId,
     });
+    const nodeId = placement.node.id;
+    // A registered `spawn:<harness>` action shadows native capacity on this node.
+    // Capacity-direct delegation (ctx.spawnAgent) bypasses the shadow so a handler
+    // that delegates cannot re-enter itself.
+    if (!args.bypassShadow) {
+        const shadow = await fetchNodeAction(args.db, args.workspaceId, nodeId, capability);
+        if (shadow && shadow.handlerProvider) {
+            if (!placement.queued)
+                await releaseNodeCapacity(args.db, args.workspaceId, nodeId);
+            const dispatched = await dispatchNodeProviderInvocation({
+                db: args.db,
+                registry: args.registry,
+                workspaceId: args.workspaceId,
+                invocationId: invocation.id,
+                nodeId,
+                providerName: shadow.handlerProvider,
+                action: capability,
+                input,
+                queue: shadow.queue,
+            });
+            return spawnResult(invocation, nodeId, dispatched);
+        }
+    }
+    const capProvider = (await capacityProviderName(args.db, args.workspaceId, nodeId, capability)) ?? DEFAULT_PROVIDER_NAME;
     const dispatched = await dispatchNodeInvocation({
         db: args.db,
         registry: args.registry,
         workspaceId: args.workspaceId,
         invocationId: invocation.id,
-        nodeId: placement.node.id,
-        action: placement.capability,
-        input: recordInput(invocation.input),
+        nodeId,
+        providerName: capProvider,
+        action: capability,
+        input,
         pending: placement.queued,
         reservationHeld: !placement.queued,
     });
+    return spawnResult(invocation, nodeId, dispatched);
+}
+/**
+ * Capacity-direct spawn used by handler-context `ctx.spawnAgent`: it targets a
+ * node's capacity executor and never re-enters action dispatch, so a
+ * `spawn:<harness>` shadow handler that delegates cannot recurse into itself.
+ */
+export async function dispatchCapacitySpawn(db, workspaceId, data, options = {}) {
+    return dispatchSpawn({
+        db,
+        registry: options.nodeConnections,
+        workspaceId,
+        data,
+        targetNodeId: data.target_node_id,
+        bypassShadow: true,
+    });
+}
+/**
+ * Invoke a node-addressed action: `POST /v1/nodes/:node/actions/:name/invoke`.
+ * Resolves capability -> provider -> socket on the given node. A `spawn:*` name
+ * with no shadow action falls through to native capacity on that node.
+ */
+export async function invokeNodeAction(db, workspaceId, nodeId, actionName, data, options = {}) {
+    if (!options.nodeConnections) {
+        throw codedError('Node dispatch is not available', 'node_dispatch_unavailable', 503);
+    }
+    const action = await fetchNodeAction(db, workspaceId, nodeId, actionName);
+    if (!action) {
+        if (isSpawnInvocation(actionName)) {
+            // Carry the harness from the URL (`spawn:<harness>`) into placement so the
+            // native fallback targets the right capacity even when the body omits it.
+            const harness = actionName.startsWith('spawn:') ? actionName.slice('spawn:'.length) : undefined;
+            const spawnInput = harness ? { ...(data.input ?? {}), capability: harness } : data.input;
+            return dispatchSpawn({ db, registry: options.nodeConnections, workspaceId, data: { ...data, input: spawnInput }, targetNodeId: nodeId });
+        }
+        if (isReleaseInvocation(actionName)) {
+            return dispatchRelease({ db, registry: options.nodeConnections, workspaceId, data });
+        }
+        throw codedError(`Action "${actionName}" not found on node`, 'action_not_found', 404);
+    }
+    if (action.availableTo && action.availableTo.length > 0) {
+        if (!data.caller_name || !action.availableTo.includes(data.caller_name)) {
+            const who = data.caller_name ? `Agent "${data.caller_name}"` : 'Caller';
+            throw codedError(`${who} is not authorized to invoke action "${actionName}"`, 'action_denied', 403);
+        }
+    }
+    const invocation = await createInvocation(db, workspaceId, action, {
+        input: data.input,
+        caller_id: data.caller_id,
+        caller_name: data.caller_name,
+    });
+    const providerName = action.handlerProvider ?? DEFAULT_PROVIDER_NAME;
+    const reservedNode = isSpawnInvocation(action.name)
+        ? await reserveNodeCapacity(db, workspaceId, nodeId)
+        : null;
+    const dispatched = await dispatchNodeProviderInvocation({
+        db,
+        registry: options.nodeConnections,
+        workspaceId,
+        invocationId: invocation.id,
+        nodeId,
+        providerName,
+        action: action.name,
+        input: recordInput(invocation.input),
+        queue: action.queue,
+        reservationHeld: !!reservedNode,
+    });
     return {
         invocation_id: invocation.id,
-        action_name: 'spawn',
+        action_name: actionName,
         handler_agent_id: null,
-        handler_node_id: placement.node.id,
-        dispatched_node_id: dispatched.accepted ? placement.node.id : null,
+        handler_node_id: nodeId,
+        dispatched_node_id: dispatched.accepted ? nodeId : null,
         input: recordInput(invocation.input),
         status: dispatched.accepted ? (dispatched.pending ? 'pending' : 'dispatched') : 'pending',
         created_at: invocation.createdAt.toISOString(),
     };
 }
 export async function invokeAction(db, workspaceId, actionName, data, options = {}) {
-    const action = await fetchAction(db, workspaceId, actionName);
+    const action = await fetchAction(db, workspaceId, actionName, options.includeNodeScoped);
     if (!action && actionName === 'spawn') {
         return dispatchSpawn({
             db,
@@ -349,14 +533,21 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
         const reservedNode = isSpawnInvocation(action.name)
             ? await reserveNodeCapacity(db, workspaceId, action.handlerNodeId)
             : null;
-        const dispatched = await dispatchNodeInvocation({
+        const providerName = action.handlerProvider ?? DEFAULT_PROVIDER_NAME;
+        // Route through the per-provider liveness gate so an offline provider fails
+        // fast unless the capability opted into queue. The synthetic `default`
+        // provider keeps the legacy queue-on-offline behavior (broker reconnect
+        // resilience), so its aliases queue and drain on reconnect as before.
+        const dispatched = await dispatchNodeProviderInvocation({
             db,
             registry: options.nodeConnections,
             workspaceId,
             invocationId: invocation.id,
             nodeId: action.handlerNodeId,
+            providerName,
             action: action.name,
             input: recordInput(invocation.input),
+            queue: action.queue || providerName === DEFAULT_PROVIDER_NAME,
             reservationHeld: !!reservedNode,
         });
         return {
@@ -382,6 +573,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
         name: agents.name,
         locationType: agents.locationType,
         locationNodeId: agents.locationNodeId,
+        providerName: agents.providerName,
     })
         .from(agents)
         .where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, action.handlerAgentId)));
@@ -399,6 +591,7 @@ export async function invokeAction(db, workspaceId, actionName, data, options =
         workspaceId,
         invocationId: invocation.id,
         nodeId: handlerAgent.locationNodeId,
+        providerName: handlerAgent.providerName,
         action: action.name,
         input: recordInput(invocation.input),
         agent: { id: handlerAgent.id, name: handlerAgent.name },
@@ -501,6 +694,7 @@ async function dispatchNodeAttempt(db, workspaceId, invocationId, nodeId, opts =
         ...stateFields,
         dispatchedNodeId: nodeId,
         spawnReservedAt: opts.reservationHeld ? new Date() : null,
+        ...(opts.actionId ? { actionId: opts.actionId } : {}),
         ...attemptFields,
     })
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
@@ -526,15 +720,20 @@ export async function markDrainedInvocationDispatched(db, workspaceId, invocatio
     return !!updated;
 }
 async function dispatchNodeInvocation(args) {
-    const connectedBefore = args.registry.isNodeConnected(args.workspaceId, args.nodeId);
-    const sent = await args.registry.sendToNode(args.workspaceId, args.nodeId, {
+    const frame = {
         v: 1,
         type: 'action.invoke',
         invocation_id: args.invocationId,
         action: args.action,
         ...(args.agent ? { agent_id: args.agent.id, agent_name: args.agent.name } : {}),
         input: toFleetWireJson(args.input),
-    });
+    };
+    const connectedBefore = args.providerName
+        ? args.registry.isProviderConnected(args.workspaceId, args.nodeId, args.providerName)
+        : args.registry.isNodeConnected(args.workspaceId, args.nodeId);
+    const sent = args.providerName
+        ? await args.registry.sendToProvider(args.workspaceId, args.nodeId, args.providerName, frame)
+        : await args.registry.sendToNode(args.workspaceId, args.nodeId, frame);
     if (!sent)
         return { accepted: false, pending: false };
     const pending = !!args.pending || !connectedBefore;
@@ -543,6 +742,7 @@ async function dispatchNodeInvocation(args) {
         retryAfterAt: args.retryAfterAt,
         reservationHeld: args.reservationHeld,
         skipIncrementAttempts: args.skipIncrementAttempts,
+        actionId: args.actionId,
     });
     return { accepted, pending };
 }
@@ -576,6 +776,7 @@ async function targetAgentForInvocation(db, invocation) {
         agentName: agents.name,
         locationType: agents.locationType,
         nodeId: agents.locationNodeId,
+        providerName: agents.providerName,
     })
         .from(actionInvocations)
         .innerJoin(actions, eq(actionInvocations.actionId, actions.id))
@@ -583,7 +784,7 @@ async function targetAgentForInvocation(db, invocation) {
         .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id)));
     if (!row || row.locationType !== 'via_node' || !row.nodeId)
         return null;
-    return { agentId: row.agentId, agentName: row.agentName, nodeId: row.nodeId };
+    return { agentId: row.agentId, agentName: row.agentName, nodeId: row.nodeId, providerName: row.providerName };
 }
 export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
     const now = new Date();
@@ -596,10 +797,12 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
         spawnReservedAt: actionInvocations.spawnReservedAt,
         dispatchedNodeId: actionInvocations.dispatchedNodeId,
         handlerNodeId: actions.handlerNodeId,
+        handlerProvider: actions.handlerProvider,
         handlerAgentId: agents.id,
         handlerAgentName: agents.name,
         handlerAgentLocationType: agents.locationType,
         handlerAgentNodeId: agents.locationNodeId,
+        handlerAgentProvider: agents.providerName,
     })
         .from(actionInvocations)
         .leftJoin(actions, eq(actionInvocations.actionId, actions.id))
@@ -617,11 +820,37 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
         const targetNodeId = targetAgent?.nodeId ?? row.handlerNodeId ?? row.dispatchedNodeId;
         if (targetNodeId !== nodeId)
             continue;
+        const input = recordInput(row.input);
+        // Resolve the target provider so the offline queue drains per provider. A
+        // spawn with a registered shadow re-dispatches to the shadow (no capacity
+        // reservation); otherwise native capacity handles it.
+        let shadowProvider = null;
+        if (isSpawnInvocation(row.actionName)) {
+            const shadow = await fetchNodeAction(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input));
+            shadowProvider = shadow?.handlerProvider ?? null;
+        }
+        const providerName = targetAgent
+            ? row.handlerAgentProvider ?? DEFAULT_PROVIDER_NAME
+            : row.handlerNodeId
+                ? row.handlerProvider ?? DEFAULT_PROVIDER_NAME
+                : shadowProvider
+                    ?? (isSpawnInvocation(row.actionName)
+                        ? (await capacityProviderName(db, workspaceId, nodeId, dispatchActionNameForInvocation(row.actionName, input))) ?? DEFAULT_PROVIDER_NAME
+                        : DEFAULT_PROVIDER_NAME);
+        const nativeSpawn = isSpawnInvocation(row.actionName) && !shadowProvider;
+        // Skip draining to a provider that is disconnected or whose heartbeat says
+        // handlers are not live. A missing provider row is a legacy single-socket
+        // node, for which connection state remains the available liveness signal.
+        if (!registry.isProviderConnected(workspaceId, nodeId, providerName))
+            continue;
+        const provider = await getProvider(db, workspaceId, nodeId, providerName);
+        const explicitlyAttached = registry.isProviderAttached?.(workspaceId, nodeId, providerName) ?? true;
+        if (provider && explicitlyAttached && !isProviderLive(provider))
+            continue;
         let reservedForDrain = false;
         try {
-            const input = recordInput(row.input);
-            let reservationHeld = isSpawnInvocation(row.actionName) && !!row.spawnReservedAt;
-            if (isSpawnInvocation(row.actionName) && !reservationHeld) {
+            let reservationHeld = nativeSpawn && !!row.spawnReservedAt;
+            if (nativeSpawn && !reservationHeld) {
                 const reserved = await reserveNodeCapacity(db, workspaceId, nodeId);
                 if (!reserved) {
                     const [node] = await db
@@ -649,6 +878,7 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
                 workspaceId,
                 invocationId: row.id,
                 nodeId,
+                providerName,
                 action: dispatchActionNameForInvocation(row.actionName, input),
                 input,
                 agent: targetAgent ? { id: targetAgent.id, name: targetAgent.name } : null,
@@ -673,9 +903,27 @@ export async function drainNodeInvocations(db, registry, workspaceId, nodeId) {
     return drained;
 }
 export async function rescheduleNodeInvocation(db, registry, invocation, opts = {}) {
-    if (invocation.dispatchedNodeId && isSpawnInvocation(invocation.actionName)) {
+    // Only release capacity the invocation actually holds. A shadowed spawn
+    // dropped its native reservation at dispatch (spawnReservedAt is null), so it
+    // must not release capacity that now belongs to another spawn.
+    if (invocation.dispatchedNodeId && isSpawnInvocation(invocation.actionName) && invocation.spawnReservedAt) {
         await releaseNodeCapacity(db, invocation.workspaceId, invocation.dispatchedNodeId);
     }
+    // A shadowed spawn stays pinned to its node: while a `spawn:<harness>` shadow
+    // is registered there, native capacity for that harness is unreachable — even
+    // across a node loss — so it never falls through to generic placement (which
+    // would silently bypass the shadow). It re-drains when the provider returns.
+    if (invocation.dispatchedNodeId && isSpawnInvocation(invocation.actionName)) {
+        const capability = dispatchActionNameForInvocation(invocation.actionName, recordInput(invocation.input));
+        const shadow = await fetchNodeAction(db, invocation.workspaceId, invocation.dispatchedNodeId, capability);
+        if (shadow) {
+            await db
+                .update(actionInvocations)
+                .set({ status: 'pending', dispatchedAt: null, spawnReservedAt: null, retryAfterAt: null })
+                .where(and(eq(actionInvocations.workspaceId, invocation.workspaceId), eq(actionInvocations.id, invocation.id), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)));
+            return false;
+        }
+    }
     const targetAgent = await targetAgentForInvocation(db, invocation);
     if (targetAgent) {
         const dispatched = await dispatchNodeInvocation({
@@ -684,6 +932,7 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
             workspaceId: invocation.workspaceId,
             invocationId: invocation.id,
             nodeId: targetAgent.nodeId,
+            providerName: targetAgent.providerName,
             action: invocation.actionName,
             input: recordInput(invocation.input),
             agent: { id: targetAgent.agentId, name: targetAgent.agentName },
@@ -703,33 +952,79 @@ export async function rescheduleNodeInvocation(db, registry, invocation, opts =
     const actionToSend = dispatchActionNameForInvocation(invocation.actionName, input);
     const baseExclude = Array.from(new Set([...attempted, ...current]));
     const candidates = opts.allowAttemptedFallback ? [baseExclude, []] : [baseExclude];
-    for (const excludeNodeIds of candidates) {
-        try {
-            const placement = await selectRetryPlacement(db, invocation, excludeNodeIds);
-            if (placement.queued) {
-                await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
-                    pending: true,
-                    retryAfterAt: opts.retryAfterAt ?? null,
-                    reservationHeld: false,
+    const providerRejectedNodeIds = new Set();
+    for (const candidateExcludes of candidates) {
+        const excludeNodeIds = new Set([...candidateExcludes, ...providerRejectedNodeIds]);
+        while (true) {
+            try {
+                const placement = await selectRetryPlacement(db, invocation, Array.from(excludeNodeIds));
+                // Resolve the owning provider before accepting node-level queued
+                // placement. A node can be live through its broker while the named
+                // provider for this action is offline, and provider queue policy must
+                // decide whether that retry may wait.
+                const target = await fetchNodeAction(db, invocation.workspaceId, placement.node.id, actionToSend);
+                const reservationHeld = isSpawnInvocation(invocation.actionName) && !placement.queued;
+                if (target?.handlerProvider) {
+                    // A spawn shadow is an action, not native capacity. claimSpawnNode
+                    // reserved capacity before discovering the shadow, so release it just
+                    // like the initial dispatch path does before provider dispatch.
+                    if (reservationHeld) {
+                        await releaseNodeCapacity(db, invocation.workspaceId, placement.node.id);
+                    }
+                    const dispatched = await dispatchNodeProviderInvocation({
+                        db,
+                        registry,
+                        workspaceId: invocation.workspaceId,
+                        invocationId: invocation.id,
+                        nodeId: placement.node.id,
+                        providerName: target.handlerProvider,
+                        action: actionToSend,
+                        actionId: target.id,
+                        input,
+                        queue: target.queue,
+                        reservationHeld: false,
+                    }, { failIfUnavailable: false });
+                    if (dispatched.providerUnavailable) {
+                        providerRejectedNodeIds.add(placement.node.id);
+                        excludeNodeIds.add(placement.node.id);
+                        continue;
+                    }
+                    return dispatched.accepted;
+                }
+                // No named action owner means native spawn/capacity or a legacy
+                // single-socket node; retain the node-level pending/default behavior.
+                if (placement.queued) {
+                    await dispatchNodeAttempt(db, invocation.workspaceId, invocation.id, placement.node.id, {
+                        pending: true,
+                        retryAfterAt: opts.retryAfterAt ?? null,
+                        reservationHeld: false,
+                        actionId: target?.id,
+                    });
+                    return true;
+                }
+                const dispatched = await dispatchNodeInvocation({
+                    db,
+                    registry,
+                    workspaceId: invocation.workspaceId,
+                    invocationId: invocation.id,
+                    nodeId: placement.node.id,
+                    action: actionToSend,
+                    actionId: target?.id,
+                    input,
+                    reservationHeld,
                 });
-                return true;
+                return dispatched.accepted;
+            }
+            catch {
+                // This candidate phase is exhausted; optionally retry attempted nodes.
+                break;
             }
-            const dispatched = await dispatchNodeInvocation({
-                db,
-                registry,
-                workspaceId: invocation.workspaceId,
-                invocationId: invocation.id,
-                nodeId: placement.node.id,
-                action: actionToSend,
-                input,
-                reservationHeld: isSpawnInvocation(invocation.actionName) && !placement.queued,
-            });
-            return dispatched.accepted;
-        }
-        catch {
-            // Try the next candidate set.
         }
     }
+    if (providerRejectedNodeIds.size > 0) {
+        await failInvocationForUnavailableProvider(db, invocation.workspaceId, invocation.id);
+        return false;
+    }
     await db
         .update(actionInvocations)
         .set({
@@ -825,6 +1120,7 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
         durationMs: actionInvocations.durationMs,
         dispatchedNodeId: actionInvocations.dispatchedNodeId,
         dispatchedAt: actionInvocations.dispatchedAt,
+        spawnReservedAt: actionInvocations.spawnReservedAt,
         attemptedNodeIds: actionInvocations.attemptedNodeIds,
         dispatchAttempts: actionInvocations.dispatchAttempts,
         createdAt: actionInvocations.createdAt,
@@ -865,7 +1161,10 @@ export async function completeNodeInvocation(db, registry, workspaceId, nodeId,
     })
         .where(and(eq(actionInvocations.workspaceId, workspaceId), eq(actionInvocations.id, invocationId), eq(actionInvocations.dispatchedNodeId, nodeId), inArray(actionInvocations.status, OPEN_INVOCATION_STATUSES)))
         .returning();
-    if (updated && isSpawnInvocation(updated.actionName) && updated.dispatchedNodeId) {
+    // Release only capacity this invocation actually reserved. A shadowed spawn
+    // holds no native reservation (spawnReservedAt is null), so releasing here
+    // would decrement capacity owned by another spawn.
+    if (updated && isSpawnInvocation(updated.actionName) && updated.dispatchedNodeId && existing.spawnReservedAt) {
         await releaseNodeCapacity(db, workspaceId, updated.dispatchedNodeId);
     }
     if (updated) {